level=info ts=2026-05-19T02:40:08.609397567Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-19T02:40:08.609417039Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-19T02:40:08.609433147Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-19T02:40:08.610126274Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-19T02:40:08.610790483Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-19T02:40:08.610865909Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.78 inf=eth0
level=info ts=2026-05-19T02:40:08.61132828Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-49d5f4bb
ts=2026-05-19T02:40:08.611376742Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-19T02:40:08.611385056Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.78 advertisePort=7946
level=debug ts=2026-05-19T02:40:08.611404206Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.78 advertisePort=7946
level=info ts=2026-05-19T02:40:08.611556427Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-19T02:40:08.615975895Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T02:40:08.615997721Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-19T02:40:08.617025505Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
ts=2026-05-19T02:40:08.617191653Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T02:40:08.617212598Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T02:40:08.617227753Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.672218ms
level=error ts=2026-05-19T02:40:08.617247746Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-19T02:40:08.617259238Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-05-19T02:40:08.617250599Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T02:40:08.617467376Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-19T02:40:08.617483373Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-19T02:40:08.617481271Z
level=debug ts=2026-05-19T02:40:08.617495641Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-19T02:40:08.617521653Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-19T02:40:08.617558439Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-19T02:40:08.617566159Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-19T02:40:08.617564721Z
level=debug ts=2026-05-19T02:40:08.617573635Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-05-19T02:40:08.619617761Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-19T02:40:08.619705012Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.78 inf=eth0
ts=2026-05-19T02:40:08.620934096Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T02:40:08.620952205Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-19T02:40:08.633862031Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-19T02:40:08.633876308Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-19T02:40:08.633886177Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-19T02:40:08.633894418Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-19T02:40:08.633897962Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-19T02:40:08.63390298Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-05-19T02:40:08.633900624Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-19T02:40:08.633921645Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-19T02:40:08.633918611Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-19T02:40:08.633917551Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-19T02:40:08.634111255Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=info ts=2026-05-19T02:40:08.634116602Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-05-19T02:40:08.634145384Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-19T02:40:08.634153929Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-19T02:40:08.634159787Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-05-19T02:40:08.634167553Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-05-19T02:40:08.634170791Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-19T02:40:08.634189295Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-19T02:40:08.634192845Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-19T02:40:08.634196178Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-19T02:40:08.634249362Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-05-19T02:40:08.634317371Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-19T02:40:08.634322829Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-19T02:40:08.634326203Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-05-19T02:40:08.634330713Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-19T02:40:08.63434374Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-19T02:40:08.634353022Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T02:40:08.63439624Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=20.671µs success=true
level=info ts=2026-05-19T02:40:08.634409799Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-19T02:40:08.634466257Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=124.849µs errors=false
level=info ts=2026-05-19T02:40:08.634473904Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-19T02:40:08.634616042Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=274.002µs errors=false
level=info ts=2026-05-19T02:40:08.63462844Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-19T02:40:08.634636449Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=294.614µs
level=info ts=2026-05-19T02:40:08.634655265Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-19T02:40:08.634750337Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=22.666767556s
level=info ts=2026-05-19T02:40:08.63478347Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-19T02:40:08.634798493Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-19T02:40:08.634827725Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-19T02:40:08.634840659Z caller=loki.go:615 msg="Loki started" startup_time=92.637932ms
level=debug ts=2026-05-19T02:40:08.634914469Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-19T02:40:08.634920145Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-19T02:40:08.634953975Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-19T02:40:09.856331653Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T02:40:09.856355007Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:40:12.274637054Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T02:40:12.274662662Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T02:40:13.635512519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:40:17.856204351Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T02:40:17.856224831Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T02:40:18.635611566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:40:23.635065218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:40:24.85502807Z caller=logging.go:137 msg="GET /ready (503) 92.81µs"
level=warn ts=2026-05-19T02:40:28.635333769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:40:30.415509583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.77:7946"
ts=2026-05-19T02:40:30.416798365Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.80:7946: dial tcp 10.128.0.80:7946: connect: connection refused"
ts=2026-05-19T02:40:30.417612899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.79:7946"
level=info ts=2026-05-19T02:40:30.418085815Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=21.800819501s
level=warn ts=2026-05-19T02:40:33.635018156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:40:34.853048936Z caller=logging.go:137 msg="GET /ready (503) 65.013µs"
level=info ts=2026-05-19T02:40:38.635010912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-19T02:40:38.635033708Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-19T02:40:38.635039115Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:40:38.635042589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:40:39.112708021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=warn ts=2026-05-19T02:40:43.635927239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:40:48.411074683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39880"
level=warn ts=2026-05-19T02:40:48.634920052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:40:53.635154362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:40:56.57042248Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T02:40:56.570472082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:40:56.576227073Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:40:58.635070919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:41:03.635711645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:41:06.027338093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:41:06.919053807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:54680"
level=info ts=2026-05-19T02:41:08.617367961Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:41:08.61759271Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:41:08.635884048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:41:08.635902846Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:41:08.635910762Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:41:08.635932939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:41:09.113669203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=warn ts=2026-05-19T02:41:13.635012305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:41:14.81749605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:41:14.824441827Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:41:18.635365771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:41:22.360009126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T02:41:23.635745005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:41:26.604947674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:41:28.635833446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:41:33.635953954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:41:35.186242376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:41:35.192786012Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T02:41:38.635352944Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:41:38.635347482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:41:38.635373332Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:41:38.635381583Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T02:41:39.115114427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=warn ts=2026-05-19T02:41:43.635089922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:41:45.989271462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T02:41:48.634903393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:41:53.635107565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:41:58.635513025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:42:00.614874828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:42:03.635163379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:42:07.355041444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-19T02:42:08.617321743Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:42:08.617327801Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:42:08.635172437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:42:08.635188566Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:42:08.635196324Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:42:08.635188284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:42:09.117055916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=warn ts=2026-05-19T02:42:13.635550976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:42:13.71704688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48328"
ts=2026-05-19T02:42:18.421720672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50028"
level=warn ts=2026-05-19T02:42:18.635657353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:42:23.635566778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:42:25.529731362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:42:25.535971361Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:42:28.635166366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:42:31.521257645Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:42:33.635295073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:42:36.033966948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-19T02:42:36.035667224Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T02:42:36.927653059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47714"
level=info ts=2026-05-19T02:42:38.635553508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:42:38.635549612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:42:38.635573957Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:42:38.635581537Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T02:42:39.119108212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=warn ts=2026-05-19T02:42:43.635172434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:42:48.425430269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40612"
level=warn ts=2026-05-19T02:42:48.635328598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:42:53.635370758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:42:58.635335757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:43:03.517550785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:43:03.634970932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:43:08.617288235Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:43:08.617623774Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:43:08.63530675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:43:08.635325759Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:43:08.635333378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:43:08.635348631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:43:09.12056299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T02:43:10.106751123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:43:10.114000534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:43:13.169762468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T02:43:13.635908212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:43:18.635579629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:43:23.635738737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:43:28.63585271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:43:33.635129604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:43:34.835940798Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T02:43:36.93186459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47186"
level=info ts=2026-05-19T02:43:38.635353241Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:43:38.635376911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:43:38.635385031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:43:38.635392317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:43:39.122991531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=warn ts=2026-05-19T02:43:43.634910734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:43:43.722185325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41064"
level=warn ts=2026-05-19T02:43:48.634927567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:43:53.635100089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:43:53.780465474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:43:53.788910247Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:43:58.635675972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:43:59.446643082Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:44:03.635770023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:44:08.61730756Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:44:08.617596784Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T02:44:08.63496448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:44:08.634981081Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:44:08.634997081Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:44:08.635004637Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T02:44:09.126782661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=warn ts=2026-05-19T02:44:13.635838464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:44:13.722963582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37344"
level=debug ts=2026-05-19T02:44:13.760694343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T02:44:18.635195096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:44:23.63558296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:44:24.908842964Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:44:28.635685157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:44:33.635751619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:44:38.635311963Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:44:38.635318234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:44:38.635334831Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:44:38.635343405Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T02:44:39.129248191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=warn ts=2026-05-19T02:44:43.634913382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:44:43.723764156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55878"
level=warn ts=2026-05-19T02:44:48.635842892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:44:51.300334886Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T02:44:51.917208672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:44:51.925238127Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:44:53.635837737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:44:58.635190349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:45:03.635102605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:45:08.354451184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-19T02:45:08.61702094Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:45:08.618062095Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:45:08.618269743Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-19T02:45:08.618312582Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976842 n=1
level=debug ts=2026-05-19T02:45:08.618351989Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976842 n=1
level=info ts=2026-05-19T02:45:08.635561919Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T02:45:08.63558649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:45:08.635599259Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:45:08.635606646Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:45:08.635611658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:45:08.635661229Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-19T02:45:08.635668455Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T02:45:08.6356811Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=8.051µs success=true
level=debug ts=2026-05-19T02:45:08.63591373Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
ts=2026-05-19T02:45:09.130256073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T02:45:12.921805454Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T02:45:13.635474145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:45:17.207887798Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=warn ts=2026-05-19T02:45:18.63568757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:45:19.412009113Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:45:21.493927152Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T02:45:23.635465643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:45:25.77922676Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-19T02:45:28.634906438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:45:30.065353061Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-19T02:45:31.980229276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:45:31.987836331Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:45:33.635429279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:45:34.350602604Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-19T02:45:36.943658156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45740"
level=info ts=2026-05-19T02:45:38.635655652Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:45:38.635678153Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:45:38.635686583Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:45:38.635701369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:45:38.63697591Z caller=checkpoint.go:372 msg="writing series" size="505 kB"
level=debug ts=2026-05-19T02:45:38.636990139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
ts=2026-05-19T02:45:39.131691833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:45:42.922004841Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-19T02:45:43.635085529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:45:43.728026647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35770"
level=debug ts=2026-05-19T02:45:47.207598254Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-19T02:45:48.447480032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46614"
level=warn ts=2026-05-19T02:45:48.635163624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:45:48.996340369Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:45:51.493661258Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-19T02:45:53.635566579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:45:55.779202789Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=warn ts=2026-05-19T02:45:58.635392702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:00.065129317Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-19T02:46:00.612013538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T02:46:03.635597825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:04.351080822Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
level=info ts=2026-05-19T02:46:08.617312142Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:46:08.617559458Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:46:08.635500838Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:46:08.635517565Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:46:08.635525439Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:46:08.635530421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:08.636595975Z caller=checkpoint.go:372 msg="writing series" size="8.1 kB"
ts=2026-05-19T02:46:09.133960708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:46:12.921942785Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-19T02:46:13.568631525Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:46:13.635609046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:17.20786927Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
ts=2026-05-19T02:46:18.450453493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60096"
level=warn ts=2026-05-19T02:46:18.635692919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:21.493353341Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-19T02:46:23.475582449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:46:23.484354972Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:46:23.635824752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:25.779077982Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-19T02:46:28.635230278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:30.06530542Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-19T02:46:33.635776941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:34.351102799Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-19T02:46:38.635859912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:46:38.635866003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:38.635882569Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-19T02:46:38.635878881Z caller=checkpoint.go:372 msg="writing series" size="510 B"
level=info ts=2026-05-19T02:46:38.635890342Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T02:46:39.136232683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:46:42.922489731Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=debug ts=2026-05-19T02:46:43.10243874Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:46:43.635525414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:47.207359784Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-19T02:46:48.634873794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:51.493218574Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-19T02:46:53.635747531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:46:55.779703567Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T02:46:58.635232291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:00.064743008Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-19T02:47:03.635865742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:04.350841528Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-19T02:47:08.617502885Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:47:08.617904303Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:47:08.635420604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:47:08.635440657Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:47:08.635448653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:47:08.635479886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:08.636547522Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-19T02:47:09.137338908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:47:12.922590963Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-19T02:47:13.635719333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:47:13.733716055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36316"
level=debug ts=2026-05-19T02:47:14.815913193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-19T02:47:14.817356132Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T02:47:16.712260779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:47:17.207318419Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-05-19T02:47:18.456678113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33654"
level=warn ts=2026-05-19T02:47:18.635851886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:21.494024026Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-19T02:47:21.494042754Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=24
level=info ts=2026-05-19T02:47:21.955903121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:47:21.963598319Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:47:21.96361812Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T02:47:21.963640796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:47:21.969283412Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:47:23.635023492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:25.779510699Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=warn ts=2026-05-19T02:47:28.635476739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:30.065044483Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=warn ts=2026-05-19T02:47:33.635355828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:34.350826284Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-19T02:47:36.953962063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:59256"
level=info ts=2026-05-19T02:47:38.635120643Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:47:38.635143997Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:47:38.635151923Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:47:38.635163995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:38.636245949Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
ts=2026-05-19T02:47:39.138644099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T02:47:40.230233423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:47:40.238516671Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:47:42.922184984Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-19T02:47:43.635449296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:47.207979469Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=debug ts=2026-05-19T02:47:48.401136553Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:47:48.635786735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:51.493975191Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-19T02:47:53.635855477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:55.779232793Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-19T02:47:58.635648643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:47:59.369415035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T02:48:00.064490503Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-19T02:48:00.952726695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:48:00.958803975Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:48:03.635582632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:04.350305587Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
ts=2026-05-19T02:48:06.955724405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39406"
level=debug ts=2026-05-19T02:48:08.617149366Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:48:08.618233047Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T02:48:08.635895229Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:48:08.635901616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:08.63591424Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:48:08.635922456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T02:48:08.635917947Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
ts=2026-05-19T02:48:09.139690931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T02:48:12.921904319Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-19T02:48:13.635479957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:48:13.736613794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33202"
level=debug ts=2026-05-19T02:48:16.243120016Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:48:17.207483464Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-19T02:48:18.635361058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:21.493915572Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-19T02:48:23.635405405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:25.77897Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-19T02:48:28.635485396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:30.066277638Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-19T02:48:30.066299169Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=16
level=warn ts=2026-05-19T02:48:33.635903135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:34.351070155Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-19T02:48:37.734315399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T02:48:38.635489743Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:48:38.635511822Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:48:38.635522696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:48:38.635551346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:38.636652632Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
ts=2026-05-19T02:48:39.14241871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T02:48:40.978730764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:48:40.986222625Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:48:42.922517324Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
level=warn ts=2026-05-19T02:48:43.635408921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:45.814601201Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:48:47.208310623Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
ts=2026-05-19T02:48:48.466891067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36370"
level=warn ts=2026-05-19T02:48:48.635512841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:51.494030253Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T02:48:53.635513974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:48:55.782940472Z caller=checkpoint.go:372 msg="writing series" size="5.6 MB"
level=debug ts=2026-05-19T02:48:55.782966769Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.2 MB" series=6
level=warn ts=2026-05-19T02:48:58.635174172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:49:00.065251232Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-19T02:49:03.635706644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:49:04.350940594Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-19T02:49:08.618066963Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:49:08.618834886Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:49:08.635404753Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:49:08.635421837Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:49:08.635429787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:49:08.635437609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:49:08.636612354Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
ts=2026-05-19T02:49:09.145972714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:49:12.921743379Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-19T02:49:13.635327989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:49:17.207839583Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-19T02:49:17.384301861Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T02:49:18.469659162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36246"
level=warn ts=2026-05-19T02:49:18.635313015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:49:21.493161266Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=warn ts=2026-05-19T02:49:23.635782642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:49:24.85225263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T02:49:25.779184374Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-19T02:49:28.635434352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:49:30.065222117Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-19T02:49:30.065240947Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-19T02:49:33.635710456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:49:34.350518574Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
ts=2026-05-19T02:49:36.962667812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33176"
level=info ts=2026-05-19T02:49:37.263070026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:49:37.271432421Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T02:49:38.635515236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:49:38.635547734Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:49:38.635555279Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:49:38.635527389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:49:38.636621018Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.7 kB" series=1
level=info ts=2026-05-19T02:49:38.641948203Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-19T02:49:38.645121207Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009322681s
ts=2026-05-19T02:49:39.148298875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T02:49:42.172285472Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:49:43.635855109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:49:48.473275627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33206"
level=warn ts=2026-05-19T02:49:48.635114953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:49:53.635800924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:49:58.635726391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:50:03.635826172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:50:08.618244376Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:50:08.618679568Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:50:08.635636059Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-19T02:50:08.635675664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:50:08.635672984Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:50:08.635702078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:50:08.635710974Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T02:50:08.635753958Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-19T02:50:08.635761697Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T02:50:08.635781492Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.896µs success=true
level=debug ts=2026-05-19T02:50:08.636017885Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
ts=2026-05-19T02:50:09.150429524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:50:12.745965482Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:50:12.922608117Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T02:50:13.635669339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:50:17.20804345Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=warn ts=2026-05-19T02:50:18.635681657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:50:19.334870612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:50:19.342313222Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:50:21.493162828Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T02:50:23.635354253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:50:24.339049508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T02:50:25.77889707Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-19T02:50:28.635553791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:50:30.064808318Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-19T02:50:33.635023991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:50:34.350415634Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
ts=2026-05-19T02:50:36.965855972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:59118"
level=info ts=2026-05-19T02:50:38.634974202Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:50:38.634979829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:50:38.634999392Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:50:38.635007584Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T02:50:38.636271241Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=debug ts=2026-05-19T02:50:38.636282774Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
ts=2026-05-19T02:50:39.153442101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:50:42.922407788Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-19T02:50:43.635107342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:50:44.39471341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:50:47.207794929Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-19T02:50:48.479723257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41154"
level=warn ts=2026-05-19T02:50:48.635289145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:50:51.494020078Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-19T02:50:53.635617838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:50:55.779252065Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-19T02:50:58.635584037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:00.06509087Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T02:51:03.634905733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:04.350917148Z caller=checkpoint.go:372 msg="writing series" size="681 kB"
level=info ts=2026-05-19T02:51:05.86859506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:51:05.876936435Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T02:51:08.617334343Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:51:08.617620537Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:51:08.634997702Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:51:08.635007289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:08.63502538Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:51:08.635033764Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T02:51:08.636145449Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-19T02:51:09.157198884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:51:12.922630474Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-19T02:51:13.635921773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:17.207896478Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-19T02:51:17.207919381Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-19T02:51:18.236640548Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T02:51:18.482781004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42410"
level=warn ts=2026-05-19T02:51:18.635530165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:20.539926019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T02:51:20.541760908Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T02:51:21.49344928Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-19T02:51:23.635577966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:25.77992278Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=warn ts=2026-05-19T02:51:28.635400125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:30.064851584Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-19T02:51:33.635615423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:34.350609761Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-19T02:51:38.634888642Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:51:38.634911729Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T02:51:38.634904706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:51:38.634920321Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T02:51:38.636009681Z caller=checkpoint.go:372 msg="writing series" size="510 B"
ts=2026-05-19T02:51:39.159767091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T02:51:42.922634271Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-19T02:51:43.635311893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:51:43.75656623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60770"
level=debug ts=2026-05-19T02:51:47.207842648Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-19T02:51:48.635589548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:49.117285025Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:51:51.493393698Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-19T02:51:53.635032161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:51:55.779899758Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-19T02:51:58.635245971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:51:59.892225214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:51:59.900043617Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:52:00.065588801Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-19T02:52:03.635399792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:52:04.350631871Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
ts=2026-05-19T02:52:06.97003002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:51166"
level=info ts=2026-05-19T02:52:08.617331445Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:52:08.617752468Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:52:08.63493772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:52:08.634959683Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T02:52:08.63495991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:52:08.634971099Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T02:52:08.636051279Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-19T02:52:09.162006681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:52:12.922543736Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-19T02:52:13.635582587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:52:14.777695981Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:52:17.207911282Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T02:52:18.635668109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:52:21.493668357Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-19T02:52:22.390718043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T02:52:23.635102484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:52:25.779555894Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=debug ts=2026-05-19T02:52:25.779578068Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=16
level=warn ts=2026-05-19T02:52:28.635648611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:52:30.065204403Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=warn ts=2026-05-19T02:52:33.635312239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:52:34.350816876Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-19T02:52:36.971466013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33218"
level=info ts=2026-05-19T02:52:38.634844298Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:52:38.634866963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:52:38.634875069Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:52:38.634884668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:52:38.637016579Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
ts=2026-05-19T02:52:39.166380073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:52:42.921767348Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-19T02:52:43.634893264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:52:43.92171375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:52:43.929178403Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:52:45.002400411Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:52:47.207397636Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T02:52:48.635594564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:52:51.494044041Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-19T02:52:53.635674622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:52:55.779100247Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-19T02:52:58.635670009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:00.065669307Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-19T02:53:03.634892147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:04.350895935Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-19T02:53:08.617577372Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:53:08.61855312Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:53:08.634976107Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:53:08.634982337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:08.63499871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:53:08.635009858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T02:53:08.636183787Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
ts=2026-05-19T02:53:09.168980889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:53:12.922513188Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-19T02:53:13.635103989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:13.984166719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T02:53:17.208435168Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-19T02:53:18.635619921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:20.650304706Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T02:53:20.650377264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="28 MB" avg_uncomp="14 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:53:20.65726172Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T02:53:21.493417983Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-19T02:53:23.635910357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:25.779333336Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-19T02:53:28.634918147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:30.065852492Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-19T02:53:30.065881873Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=15
level=warn ts=2026-05-19T02:53:33.635816882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:34.350823373Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
ts=2026-05-19T02:53:36.974616993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:52792"
level=info ts=2026-05-19T02:53:38.299823424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:53:38.307386789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:53:38.307407575Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T02:53:38.307428339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:53:38.313484238Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T02:53:38.635206337Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:53:38.635210149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:38.635228332Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:53:38.635236751Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T02:53:38.636425263Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
ts=2026-05-19T02:53:39.169794954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T02:53:40.458283674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="28 MB" avg_uncomp="14 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:53:40.465049829Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T02:53:42.922796458Z caller=checkpoint.go:372 msg="writing series" size="346 kB"
level=warn ts=2026-05-19T02:53:43.635267286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:47.208420757Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-19T02:53:48.635789833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:53:50.252019928Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:53:51.493682795Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T02:53:53.635587054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:53:55.473445946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:53:55.481854499Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:53:55.783967305Z caller=checkpoint.go:372 msg="writing series" size="6.5 MB"
level=debug ts=2026-05-19T02:53:55.783988041Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.0 MB" series=6
level=debug ts=2026-05-19T02:53:58.562439439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T02:53:58.635323229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:54:00.065427619Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-19T02:54:03.635777093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:54:04.35069788Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
ts=2026-05-19T02:54:06.976198642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:35762"
level=info ts=2026-05-19T02:54:08.61737865Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:54:08.617659945Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:54:08.634935712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:54:08.634955633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:54:08.634963957Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:54:08.634957785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:54:08.636123353Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
ts=2026-05-19T02:54:09.172965803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:54:12.922273905Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-19T02:54:13.635363768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:54:15.693047402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="28 MB" avg_uncomp="14 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:54:15.699945643Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T02:54:17.208067611Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
ts=2026-05-19T02:54:18.504066817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48640"
level=warn ts=2026-05-19T02:54:18.635885796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:54:20.176378037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:54:20.184049497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:54:21.493711337Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-19T02:54:23.118391959Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:54:23.635937659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:54:25.779219946Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-19T02:54:28.635583561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:54:30.06533827Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-19T02:54:30.065364738Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-19T02:54:33.63534147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:54:34.350776736Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-19T02:54:38.635646333Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:54:38.635671859Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:54:38.635681299Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:54:38.63568772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:54:38.63675607Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.7 kB" series=1
level=info ts=2026-05-19T02:54:38.63947319Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-19T02:54:38.642036802Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006123591s
ts=2026-05-19T02:54:39.176797863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:54:43.353835442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T02:54:43.635732583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:54:48.507156415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41304"
level=warn ts=2026-05-19T02:54:48.63497809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:54:53.635010049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:54:54.092923474Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T02:54:57.340231622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="28 MB" avg_uncomp="14 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:54:57.349495867Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T02:54:58.635131067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T02:55:03.635655174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:55:08.617815695Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:55:08.618920675Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:55:08.635140204Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T02:55:08.635184371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:55:08.63520286Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T02:55:08.635196005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:55:08.635210635Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T02:55:08.635252289Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-19T02:55:08.635270084Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T02:55:08.635286451Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.195µs success=true
level=debug ts=2026-05-19T02:55:08.635600152Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
ts=2026-05-19T02:55:09.179929871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T02:55:09.614926261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:55:09.623963454Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:55:12.439285323Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T02:55:13.635586076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:55:16.241623981Z caller=checkpoint.go:372 msg="writing series" size="426 kB"
level=warn ts=2026-05-19T02:55:18.635721502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:55:20.044394914Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T02:55:23.6354593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:55:23.847407196Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=debug ts=2026-05-19T02:55:27.650202744Z caller=checkpoint.go:372 msg="writing series" size="371 kB"
level=debug ts=2026-05-19T02:55:27.650224716Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-19T02:55:28.635007891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:55:28.693316166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:55:31.452723529Z caller=checkpoint.go:372 msg="writing series" size="325 kB"
level=warn ts=2026-05-19T02:55:33.635600854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:55:35.255768342Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
level=info ts=2026-05-19T02:55:38.635246118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:55:38.635279761Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:55:38.635288591Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:55:38.635306663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:55:38.810969249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T02:55:38.812840841Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T02:55:39.059068236Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-19T02:55:39.183661288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T02:55:39.743768721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="28 MB" avg_uncomp="14 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:55:39.750736135Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T02:55:42.860880253Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T02:55:43.635653002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:55:43.777990055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52420"
level=debug ts=2026-05-19T02:55:46.66405559Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-19T02:55:48.635687095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:55:50.467456691Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-19T02:55:53.635858318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:55:54.269687734Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=debug ts=2026-05-19T02:55:54.269712564Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-05-19T02:55:58.072599774Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T02:55:58.635377837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:01.879274463Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-19T02:56:01.879297976Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.2 MB" series=2
level=info ts=2026-05-19T02:56:02.365331158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:56:02.373127598Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:56:03.635591626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:04.223837768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:56:05.678778037Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T02:56:08.617298693Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:56:08.617932988Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:56:08.634950064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:56:08.634955424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:08.634972988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:56:08.634981733Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T02:56:09.185921256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T02:56:09.48151548Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-19T02:56:13.284047304Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T02:56:13.635635674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:17.087090989Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-19T02:56:18.635461401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:20.889427906Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-19T02:56:23.635154501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:24.692229493Z caller=checkpoint.go:372 msg="writing series" size="299 kB"
level=debug ts=2026-05-19T02:56:28.495396063Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-19T02:56:28.635681223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:28.660888178Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:56:32.298461633Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-19T02:56:33.635163127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:56:33.693734812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:56:33.701689961Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T02:56:36.100703054Z caller=checkpoint.go:372 msg="writing series" size="510 B"
level=debug ts=2026-05-19T02:56:36.111837642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T02:56:38.635507218Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:56:38.635552138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:56:38.635560252Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:56:38.635581649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:56:39.187117663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:56:39.9040219Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-19T02:56:43.635861582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:43.705995164Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-19T02:56:47.509309671Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-05-19T02:56:48.521665563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60260"
level=warn ts=2026-05-19T02:56:48.634999595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:51.312042202Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-19T02:56:53.635761261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:55.115207677Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=info ts=2026-05-19T02:56:57.35373676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:56:57.36282188Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T02:56:58.635645714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:56:58.917510882Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-19T02:57:00.118646009Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:57:02.721013545Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-19T02:57:03.635577455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:06.523107612Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-19T02:57:06.991073131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:59880"
level=info ts=2026-05-19T02:57:08.617295957Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:57:08.618098564Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:57:08.635218723Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:57:08.635240134Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T02:57:08.635234452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:57:08.635247454Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T02:57:09.18952717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:57:10.326436293Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-19T02:57:10.326464329Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=18
level=warn ts=2026-05-19T02:57:13.635742387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:14.129314093Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-19T02:57:17.932440773Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
ts=2026-05-19T02:57:18.525239755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53942"
level=warn ts=2026-05-19T02:57:18.635477178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:57:19.401605068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:57:19.410273505Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T02:57:21.73521261Z caller=checkpoint.go:372 msg="writing series" size="342 kB"
level=warn ts=2026-05-19T02:57:23.635066774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:25.53791004Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=debug ts=2026-05-19T02:57:27.681090318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:57:28.635631602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:29.340679063Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-19T02:57:33.143388877Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=debug ts=2026-05-19T02:57:33.143414218Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-19T02:57:33.635517404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:35.900653625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T02:57:36.945752963Z caller=checkpoint.go:372 msg="writing series" size="393 kB"
ts=2026-05-19T02:57:36.992214052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:41554"
level=info ts=2026-05-19T02:57:38.635484456Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:57:38.635498016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:38.635510975Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:57:38.635518645Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T02:57:39.192714843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:57:40.748503193Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T02:57:43.63561925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:44.551219097Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-19T02:57:45.658751476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:57:45.667090791Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:57:48.354039229Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-19T02:57:48.635480732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:52.157170474Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-19T02:57:53.635457865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:55.959935879Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-19T02:57:58.635522725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:57:59.483282708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:57:59.763301537Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-19T02:58:03.565127405Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-19T02:58:03.635396052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:58:07.368892557Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-19T02:58:08.617978817Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:58:08.618812941Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:58:08.635797185Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:58:08.635818955Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:58:08.635827183Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:58:08.635843667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:58:09.196669243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:58:11.171549248Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=info ts=2026-05-19T02:58:12.753513345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:58:12.763963882Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T02:58:13.635053884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:58:14.974077272Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-19T02:58:18.635116486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:58:18.776662281Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=debug ts=2026-05-19T02:58:21.298682433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T02:58:22.579426644Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-19T02:58:23.63560664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:58:26.382952179Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=debug ts=2026-05-19T02:58:26.382977599Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-05-19T02:58:28.635495526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:58:30.186512162Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-19T02:58:30.186557714Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=warn ts=2026-05-19T02:58:33.635222034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:58:33.988199753Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-19T02:58:34.210255932Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T02:58:35.646386896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:58:35.654295552Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:58:37.790955658Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-19T02:58:38.635026483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:58:38.635033111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:58:38.63505787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:58:38.635066842Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T02:58:39.199517008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T02:58:41.59371485Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=warn ts=2026-05-19T02:58:43.635595369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:58:43.786615045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36418"
level=debug ts=2026-05-19T02:58:45.396136619Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-19T02:58:48.634908475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:58:49.199425753Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-19T02:58:53.001729166Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T02:58:53.634923724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:58:55.797930161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T02:58:56.810401184Z caller=checkpoint.go:372 msg="writing series" size="7.2 MB"
level=debug ts=2026-05-19T02:58:56.810423571Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.0 MB" series=7
level=warn ts=2026-05-19T02:58:58.634960041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:58:59.094970734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:58:59.103326287Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T02:58:59.416421974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T02:59:00.60738558Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=warn ts=2026-05-19T02:59:03.635477712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:59:04.411003835Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=debug ts=2026-05-19T02:59:08.213502534Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=info ts=2026-05-19T02:59:08.617288942Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T02:59:08.617784684Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T02:59:08.635505654Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T02:59:08.635548104Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:59:08.635556698Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T02:59:08.635601347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:59:09.200506433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T02:59:12.01631499Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-19T02:59:13.635448701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:59:15.819182798Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-19T02:59:18.635783799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:59:19.622449889Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-05-19T02:59:23.424805908Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-19T02:59:23.63586857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:59:27.164809549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T02:59:27.173629982Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T02:59:27.228035356Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-19T02:59:27.228057172Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-19T02:59:28.545735023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:59:28.635873477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:59:31.030685588Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-19T02:59:33.635620395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:59:34.833123273Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-19T02:59:38.634984199Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T02:59:38.634998973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:59:38.635016973Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T02:59:38.635025884Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T02:59:38.636080198Z caller=checkpoint.go:384 msg="flushing series" totalSize="129 kB" series=2
level=info ts=2026-05-19T02:59:38.638206233Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-19T02:59:38.643878008Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008467096s
ts=2026-05-19T02:59:39.202388564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=warn ts=2026-05-19T02:59:43.635939786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T02:59:43.789047289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43116"
level=debug ts=2026-05-19T02:59:44.161350881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T02:59:44.163058345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T02:59:48.635284402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T02:59:48.956866942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:59:48.963697893Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:59:48.963724961Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T02:59:48.963754978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T02:59:48.970355258Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T02:59:53.635232342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T02:59:53.894512129Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T02:59:58.635227579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:00:03.635627365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:00:07.000200178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:44360"
level=info ts=2026-05-19T03:00:08.61821609Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:00:08.618505139Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-19T03:00:08.618588118Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976843 n=1
level=debug ts=2026-05-19T03:00:08.618646948Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976843 n=1
level=debug ts=2026-05-19T03:00:08.618986692Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:00:08.634766399Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T03:00:08.634840973Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:00:08.634861396Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:00:08.634868732Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:00:08.634869989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:00:08.634878845Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-19T03:00:08.634888672Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:00:08.634909285Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.359µs success=true
level=debug ts=2026-05-19T03:00:08.635341478Z caller=checkpoint.go:372 msg="writing series" size="412 kB"
level=info ts=2026-05-19T03:00:08.871012461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:00:08.879523834Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-19T03:00:09.203811364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:00:12.438636852Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-19T03:00:12.774598311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:00:12.782511984Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:00:12.782558333Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:00:12.782584583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:00:12.790172288Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:00:13.635327084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:00:16.241732217Z caller=checkpoint.go:372 msg="writing series" size="490 kB"
ts=2026-05-19T03:00:18.546393868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35254"
level=warn ts=2026-05-19T03:00:18.635579909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:00:20.0439928Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-19T03:00:22.863487607Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:00:23.635126107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:00:23.847423612Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-19T03:00:27.329102958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:00:27.337581191Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:00:27.649794565Z caller=checkpoint.go:372 msg="writing series" size="396 kB"
level=debug ts=2026-05-19T03:00:27.649818618Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-19T03:00:28.635411701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:00:31.452952059Z caller=checkpoint.go:372 msg="writing series" size="406 kB"
level=warn ts=2026-05-19T03:00:33.63535653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:00:35.255654162Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=debug ts=2026-05-19T03:00:37.698106133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:00:38.634937905Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:00:38.634964336Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:00:38.634972641Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:00:38.634988662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:00:39.058185321Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-19T03:00:39.206971028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:00:42.86074333Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T03:00:43.63558635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:00:44.233796939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:00:44.241557926Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:00:46.663623417Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-19T03:00:48.635573468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:00:50.466391599Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-19T03:00:50.466434414Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-19T03:00:53.634894663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:00:54.269164925Z caller=checkpoint.go:372 msg="writing series" size="414 kB"
level=debug ts=2026-05-19T03:00:56.354895855Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:00:58.071756808Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T03:00:58.635010104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:01.881907407Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-19T03:01:01.881932275Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=3
level=warn ts=2026-05-19T03:01:03.635215182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:05.678341076Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:01:06.16567548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:01:06.173332787Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:01:08.617365073Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:01:08.617678175Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:01:08.635346283Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:01:08.635369583Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:01:08.635359611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:01:08.635377819Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:01:09.209675005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:01:09.480515943Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-19T03:01:13.283103782Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:01:13.635734492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:17.086396343Z caller=checkpoint.go:372 msg="writing series" size="417 kB"
ts=2026-05-19T03:01:18.553653749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54526"
level=warn ts=2026-05-19T03:01:18.63559784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:19.6164074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:01:20.888951888Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-19T03:01:23.635827304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:24.692412998Z caller=checkpoint.go:372 msg="writing series" size="315 kB"
level=debug ts=2026-05-19T03:01:28.495057955Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-19T03:01:28.635668135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:31.350290072Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:01:32.297778844Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-05-19T03:01:33.635202068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:36.100332085Z caller=checkpoint.go:372 msg="writing series" size="510 B"
level=info ts=2026-05-19T03:01:36.367146858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:01:36.374932529Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-19T03:01:37.004300331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:50386"
level=info ts=2026-05-19T03:01:38.635318728Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:01:38.635340818Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:01:38.635347977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:01:38.635364568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:01:39.211118077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:01:39.902968078Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-19T03:01:43.635190307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:43.706372163Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-19T03:01:43.70638978Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-05-19T03:01:47.509044339Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-05-19T03:01:48.635062718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:51.31232152Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-19T03:01:53.635135545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:55.114142287Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-19T03:01:58.635274389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:01:58.917505509Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-19T03:02:01.988860435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:02:01.997175935Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:02:02.720443919Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-19T03:02:03.635451598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:05.373113591Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:02:06.523293279Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-19T03:02:06.865138719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:02:08.617362068Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:02:08.617777001Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:02:08.635565487Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:02:08.635574479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:08.635584365Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:02:08.635593911Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:02:09.213210723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:02:10.326336631Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-19T03:02:13.635888056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:14.128688338Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-19T03:02:17.931949531Z caller=checkpoint.go:372 msg="writing series" size="368 kB"
level=warn ts=2026-05-19T03:02:18.635327013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:21.734807726Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-19T03:02:23.635076581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:25.537675931Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=debug ts=2026-05-19T03:02:25.537697375Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=11
level=warn ts=2026-05-19T03:02:28.635405701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:02:29.094018586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:02:29.103156544Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:02:29.339826312Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=debug ts=2026-05-19T03:02:33.143269481Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=warn ts=2026-05-19T03:02:33.63587949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:36.946098013Z caller=checkpoint.go:372 msg="writing series" size="503 kB"
level=info ts=2026-05-19T03:02:38.635301575Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:02:38.635323871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:02:38.635334562Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:02:38.635337667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:02:39.215716218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:02:39.32157169Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:02:40.748075633Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T03:02:43.635776994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:44.551489338Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-19T03:02:44.796662479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:02:48.353637531Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-19T03:02:48.63538509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:52.15673419Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-19T03:02:53.635585663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:55.959356103Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-19T03:02:55.959380125Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-19T03:02:58.634889943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:02:59.762551851Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T03:03:01.438892271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:03:01.447409679Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:03:03.565407807Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-19T03:03:03.635388437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:03:05.974015019Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:03:07.367795296Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-19T03:03:08.617622551Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:03:08.61808536Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:03:08.635394451Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:03:08.635396739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:03:08.635410539Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:03:08.635418318Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:03:09.216488269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:03:11.170929229Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-19T03:03:13.635372223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:03:13.806301586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49536"
level=debug ts=2026-05-19T03:03:14.974280455Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=warn ts=2026-05-19T03:03:18.635393763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:03:18.776778545Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=debug ts=2026-05-19T03:03:22.579595549Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-19T03:03:23.6350965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:03:26.382360974Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-19T03:03:26.441642173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:03:26.449810001Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:03:28.635352752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:03:29.714267689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:03:29.715792816Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:03:30.185799066Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-19T03:03:30.185820221Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=9
level=warn ts=2026-05-19T03:03:33.635393917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:03:33.987769359Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-19T03:03:34.896380349Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:03:37.790257985Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=info ts=2026-05-19T03:03:38.635817623Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:03:38.635839247Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:03:38.635848905Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:03:38.635850002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:03:39.21885536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:03:41.593324537Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-19T03:03:43.343738094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:03:43.350901896Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:03:43.635253604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:03:45.396328612Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-19T03:03:48.635032598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:03:49.199462015Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=debug ts=2026-05-19T03:03:53.001817361Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T03:03:53.635670791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:03:56.808117495Z caller=checkpoint.go:372 msg="writing series" size="7.6 MB"
level=debug ts=2026-05-19T03:03:56.8081385Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.4 MB" series=7
level=warn ts=2026-05-19T03:03:58.635231023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:04:00.607297394Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=debug ts=2026-05-19T03:04:02.448957735Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:04:03.635427335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:04:04.410628081Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=debug ts=2026-05-19T03:04:08.213465995Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=info ts=2026-05-19T03:04:08.617653463Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:04:08.618443182Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T03:04:08.634951374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:04:08.63494155Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:04:08.634972388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:04:08.634982993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:04:09.220569098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:04:12.015963277Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-19T03:04:13.63488811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:04:15.016246515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:04:15.02461192Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:04:15.818738Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
ts=2026-05-19T03:04:18.573516671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58374"
level=warn ts=2026-05-19T03:04:18.63566267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:04:19.621226953Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=debug ts=2026-05-19T03:04:23.424745563Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-19T03:04:23.635801648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:04:24.555956748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:04:27.227449858Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-19T03:04:27.227471843Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-19T03:04:28.635721169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:04:31.03059132Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=debug ts=2026-05-19T03:04:31.147797277Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:04:31.948936505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:04:31.955183352Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:04:33.635544229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:04:34.833132506Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=debug ts=2026-05-19T03:04:38.635140818Z caller=checkpoint.go:384 msg="flushing series" totalSize="321 kB" series=2
level=warn ts=2026-05-19T03:04:38.635155164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:04:38.635173851Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:04:38.635182963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:04:38.635190216Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T03:04:38.638951986Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-19T03:04:38.642897083Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007833061s
ts=2026-05-19T03:04:39.221815976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=warn ts=2026-05-19T03:04:43.635238798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:04:43.81248566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37596"
ts=2026-05-19T03:04:48.577172413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58202"
level=warn ts=2026-05-19T03:04:48.635480639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:04:53.635568724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:04:55.924840998Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:04:58.635899664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:05:03.635047203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:05:08.503127318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:05:08.510894626Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:05:08.617293503Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:05:08.617724774Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:05:08.634974123Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:05:08.634977135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:05:08.634980852Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-19T03:05:08.634991104Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:05:08.634999136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T03:05:08.635234591Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-19T03:05:08.63524574Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:05:08.635265716Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.814µs success=true
level=debug ts=2026-05-19T03:05:08.636029273Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
ts=2026-05-19T03:05:09.222766401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:05:12.438972095Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T03:05:13.635836814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:05:16.241700925Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-19T03:05:16.241731903Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-05-19T03:05:18.635859612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:05:18.957371251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:05:20.044405123Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T03:05:23.336213423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:05:23.343862704Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:05:23.635883691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:05:23.847599667Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-19T03:05:27.650672658Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=warn ts=2026-05-19T03:05:28.635873147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:05:29.748795732Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:05:31.453407252Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=warn ts=2026-05-19T03:05:33.635577745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:05:35.255768186Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=debug ts=2026-05-19T03:05:35.255795048Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-19T03:05:38.635861915Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:05:38.635865895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:05:38.63588436Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:05:38.635893522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:05:39.059094828Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-19T03:05:39.224011259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:05:42.861034761Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T03:05:43.635725015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:05:43.813829877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34614"
level=debug ts=2026-05-19T03:05:46.664602602Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-19T03:05:48.583346474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54810"
level=warn ts=2026-05-19T03:05:48.635702881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:05:49.263793793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="100 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:05:49.271190161Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:05:50.467101309Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-19T03:05:53.635198199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:05:54.26974127Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=debug ts=2026-05-19T03:05:58.072420802Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T03:05:58.635155441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:00.222396598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:06:01.883180457Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-19T03:06:01.883202557Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=7
level=warn ts=2026-05-19T03:06:03.635063797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:05.678037329Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:06:07.747386105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:06:07.754918245Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:06:08.61745767Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:06:08.617900643Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:06:08.635444585Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:06:08.635463063Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:06:08.635456154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:06:08.635470133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:06:09.226600499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:06:09.480682452Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=debug ts=2026-05-19T03:06:13.284088046Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:06:13.635852607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:17.08651819Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=warn ts=2026-05-19T03:06:18.635156476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:20.12025653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:06:20.889284458Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-19T03:06:23.635098182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:24.692648117Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-19T03:06:28.495076382Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-19T03:06:28.635424364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:32.298475947Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-19T03:06:33.634903683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:35.345334225Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:06:36.100500105Z caller=checkpoint.go:372 msg="writing series" size="510 B"
ts=2026-05-19T03:06:37.026848638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:46476"
level=info ts=2026-05-19T03:06:38.635565554Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:06:38.635589626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:06:38.635597083Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:06:38.635598371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:06:39.229051788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:06:39.90334476Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-19T03:06:43.635322396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:43.706586303Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
ts=2026-05-19T03:06:43.819170768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39480"
level=debug ts=2026-05-19T03:06:47.509628428Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-19T03:06:48.63586173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:06:49.255961837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="100 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:06:49.264254Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:06:49.264275289Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:06:49.264300859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="100 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:06:49.270014668Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:06:51.312336179Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-19T03:06:51.312359376Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-19T03:06:53.635046725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:55.1146132Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-19T03:06:58.63573357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:06:58.917611183Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=debug ts=2026-05-19T03:07:02.721046394Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-19T03:07:03.635763462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:07:03.737090784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:07:03.745257757Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:07:03.745282288Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:07:03.745305244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:07:03.751421223Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:07:06.523511768Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-19T03:07:07.511407947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="100 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:07:07.517575945Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:07:08.296036427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:07:08.617094947Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:07:08.618153708Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T03:07:08.635405048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:07:08.635407948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:07:08.63542331Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:07:08.635430939Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:07:09.231155622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:07:10.325912051Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=debug ts=2026-05-19T03:07:11.149219375Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:07:13.635144393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:07:13.820675749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60788"
level=debug ts=2026-05-19T03:07:14.129230366Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-19T03:07:17.931933986Z caller=checkpoint.go:372 msg="writing series" size="275 kB"
level=warn ts=2026-05-19T03:07:18.634931149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:07:19.881173625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:07:19.89170753Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:07:21.734467166Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
level=warn ts=2026-05-19T03:07:23.635389972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:07:25.537675461Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-05-19T03:07:25.537699962Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-19T03:07:28.635451899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:07:29.3404801Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-19T03:07:33.143564915Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-19T03:07:33.635721048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:07:36.94610134Z caller=checkpoint.go:372 msg="writing series" size="485 kB"
ts=2026-05-19T03:07:37.030013503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:46370"
level=info ts=2026-05-19T03:07:38.635012883Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:07:38.635017722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:07:38.63503866Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:07:38.635048387Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:07:39.233322263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:07:39.591670538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="100 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:07:39.599076018Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:07:40.748280884Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T03:07:43.635114733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:07:44.551581089Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-19T03:07:45.402098849Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:07:48.354152559Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
ts=2026-05-19T03:07:48.597063697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39342"
level=warn ts=2026-05-19T03:07:48.63564715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:07:52.156826815Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-19T03:07:53.635793615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:07:55.96032228Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-19T03:07:57.470208162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:07:57.479453964Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:07:58.635356583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:07:59.762978348Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-19T03:08:01.805507914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:08:01.807146299Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:08:03.565578081Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-19T03:08:03.63551795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:08:07.368067447Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-19T03:08:08.617309166Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:08:08.61766916Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:08:08.634902804Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:08:08.634906956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:08:08.63492027Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:08:08.634929218Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:08:09.234697626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:08:11.171245878Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-19T03:08:13.63577465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:08:14.974396554Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=debug ts=2026-05-19T03:08:14.974416917Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-19T03:08:18.635777591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:08:18.777068756Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=debug ts=2026-05-19T03:08:20.542805703Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:08:22.579763561Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-19T03:08:22.715474615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="100 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:08:22.724137428Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:08:23.635546483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:08:26.383065697Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-05-19T03:08:28.635357189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:08:30.186216898Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-19T03:08:30.186239432Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=4
level=warn ts=2026-05-19T03:08:33.635128582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:08:33.988313223Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-19T03:08:37.791353646Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-19T03:08:38.635364193Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:08:38.635389852Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:08:38.635378038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:08:38.635398112Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:08:39.236907935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:08:41.593568511Z caller=checkpoint.go:372 msg="writing series" size="307 kB"
level=debug ts=2026-05-19T03:08:43.144236449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:08:43.635370041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:08:45.397042338Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-19T03:08:48.635444652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:08:49.199594002Z caller=checkpoint.go:372 msg="writing series" size="824 kB"
level=debug ts=2026-05-19T03:08:49.19962146Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-19T03:08:52.574292565Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:08:53.002513842Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T03:08:53.635924023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:08:56.579223864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:08:56.586997906Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:08:56.80964729Z caller=checkpoint.go:372 msg="writing series" size="8.4 MB"
level=debug ts=2026-05-19T03:08:56.809667163Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.4 MB" series=2
level=warn ts=2026-05-19T03:08:58.635728327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:09:00.607394123Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-19T03:09:03.635607132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:09:04.410794872Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
ts=2026-05-19T03:09:07.036793293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:35204"
level=debug ts=2026-05-19T03:09:08.213156837Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=info ts=2026-05-19T03:09:08.618109206Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:09:08.618658272Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:09:08.635401473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:09:08.635406007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:09:08.635418388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:09:08.635426684Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:09:09.240030905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:09:12.016403995Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-19T03:09:13.635420869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:09:13.716665217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="100 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:09:13.723739092Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-19T03:09:13.830851954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59998"
level=debug ts=2026-05-19T03:09:15.818980801Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
ts=2026-05-19T03:09:18.607430036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54302"
level=warn ts=2026-05-19T03:09:18.6348727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:09:19.622095845Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-19T03:09:20.338820201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:09:21.935793316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:09:23.424910362Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-19T03:09:23.635857413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:09:27.227788559Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-19T03:09:27.22781169Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-19T03:09:28.635471664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:09:31.030557729Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-19T03:09:33.635195195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:09:34.832867508Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-19T03:09:38.63484325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:09:38.634866532Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:09:38.634873749Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:09:38.63486832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:09:38.635874247Z caller=checkpoint.go:384 msg="flushing series" totalSize="321 kB" series=2
level=info ts=2026-05-19T03:09:38.640877821Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-19T03:09:38.644518091Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009117927s
ts=2026-05-19T03:09:39.243017431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=warn ts=2026-05-19T03:09:43.634903683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:09:48.635300271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:09:52.890300653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:09:52.899786857Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:09:53.635263531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:09:57.691020709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="100 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:09:57.697493944Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:09:57.829087176Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:09:58.635060007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:10:03.635475377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:10:08.618021393Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:10:08.618621404Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:10:08.63483587Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T03:10:08.634864374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:10:08.634879455Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:10:08.634887232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:10:08.634889951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:10:08.634954313Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-19T03:10:08.634962085Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:10:08.634976961Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.417µs success=true
level=debug ts=2026-05-19T03:10:08.635432792Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
ts=2026-05-19T03:10:09.245483697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:10:12.438930336Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T03:10:13.634879073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:10:16.242087659Z caller=checkpoint.go:372 msg="writing series" size="794 kB"
level=debug ts=2026-05-19T03:10:16.242109596Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=debug ts=2026-05-19T03:10:16.54247263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:10:18.614833537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49828"
level=warn ts=2026-05-19T03:10:18.635622311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:10:20.044563931Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T03:10:23.635833491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:10:23.846489381Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-19T03:10:26.502907715Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:10:27.649666009Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=warn ts=2026-05-19T03:10:28.63569255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:10:31.452828426Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
level=warn ts=2026-05-19T03:10:33.634920368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:10:35.256142322Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=debug ts=2026-05-19T03:10:35.256163865Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-19T03:10:36.434043547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:10:36.441282279Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-19T03:10:37.043145861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55110"
level=info ts=2026-05-19T03:10:38.635665855Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:10:38.635665494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:10:38.635686689Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:10:38.63569447Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:10:39.057969238Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
ts=2026-05-19T03:10:39.246908618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:10:42.86108573Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T03:10:43.635829511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:10:45.123034628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="100 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:10:45.129353572Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:10:46.66355667Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-19T03:10:48.617552662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49446"
level=warn ts=2026-05-19T03:10:48.635354142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:10:50.466620687Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-19T03:10:50.78864428Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:10:53.63507571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:10:54.269660614Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=debug ts=2026-05-19T03:10:58.07275633Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T03:10:58.635806212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:01.884946151Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-19T03:11:01.884982224Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=7
level=debug ts=2026-05-19T03:11:02.881344365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:11:03.63549353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:05.677466182Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:11:08.617551925Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:11:08.617995391Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T03:11:08.635419938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:11:08.635429877Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:11:08.635445985Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:11:08.635453173Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:11:09.247770803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:11:09.480989853Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-19T03:11:13.283903634Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:11:13.635314012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:11:13.837699817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51202"
level=debug ts=2026-05-19T03:11:16.019601755Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:11:17.086587185Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
ts=2026-05-19T03:11:18.62056624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59172"
level=warn ts=2026-05-19T03:11:18.634887931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:20.889518552Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-19T03:11:23.63529967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:24.692553773Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-19T03:11:26.723919665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:11:26.733850612Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:11:28.495087126Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-19T03:11:28.635182725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:32.297627764Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=warn ts=2026-05-19T03:11:33.63490145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:36.100319752Z caller=checkpoint.go:372 msg="writing series" size="510 B"
ts=2026-05-19T03:11:37.048176654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47254"
level=info ts=2026-05-19T03:11:38.635684968Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:11:38.635706964Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:11:38.635714354Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:11:38.635767475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:11:39.24891448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:11:39.903641439Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-19T03:11:42.079317292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:11:42.086632408Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:11:43.635512167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:43.7056587Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-19T03:11:47.509043329Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-19T03:11:48.634933327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:49.834946114Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:11:51.311904637Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=debug ts=2026-05-19T03:11:51.311926306Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-19T03:11:53.635732218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:53.826833763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:11:53.828584742Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:11:55.115091157Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-19T03:11:58.635825761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:11:58.916875823Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=debug ts=2026-05-19T03:12:02.719938255Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-19T03:12:03.63494473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:12:06.522928527Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-19T03:12:08.618091257Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:12:08.618849144Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:12:08.635445133Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:12:08.635462305Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:12:08.635474051Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:12:08.635485087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:12:09.251159112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:12:10.326339878Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=warn ts=2026-05-19T03:12:13.635149838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:12:14.128426948Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-19T03:12:17.932076226Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=warn ts=2026-05-19T03:12:18.635653235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:12:21.734850393Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-19T03:12:21.734873442Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-19T03:12:23.635832958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:12:24.360112137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:12:24.368472373Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:12:25.537483462Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-05-19T03:12:25.653061843Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:12:28.634934367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:12:29.339708497Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=debug ts=2026-05-19T03:12:31.069556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:12:33.143313202Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-19T03:12:33.635248007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:12:36.946228866Z caller=checkpoint.go:372 msg="writing series" size="540 kB"
ts=2026-05-19T03:12:37.053635398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:51450"
level=warn ts=2026-05-19T03:12:38.635067028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:12:38.635065328Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:12:38.63508817Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:12:38.635095826Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:12:39.253718049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:12:39.426861468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:12:39.436292425Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:12:40.74788353Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T03:12:43.635936251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:12:44.551384884Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-19T03:12:48.353657069Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-19T03:12:48.635467145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:12:52.157242532Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-19T03:12:53.635417001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:12:55.959344691Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=debug ts=2026-05-19T03:12:55.959369633Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-05-19T03:12:58.312203608Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:12:58.635628099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:12:59.762743605Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-19T03:13:03.565135033Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-19T03:13:03.635924641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:13:07.055381195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:41428"
level=debug ts=2026-05-19T03:13:07.367983337Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-05-19T03:13:08.616941711Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:13:08.617993761Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T03:13:08.635756706Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:13:08.635775395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:13:08.635783696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:13:08.635799368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:13:08.965067693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:13:09.255067005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:13:11.170863257Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-19T03:13:13.635524217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:13:14.973306431Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-19T03:13:15.372401981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:13:15.380709802Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:13:18.634969663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:13:18.776272588Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-19T03:13:20.654113568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:13:20.661409384Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:13:20.661436846Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:13:20.661465449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:13:20.668025056Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:13:22.57910633Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-19T03:13:23.635211786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:13:26.382378511Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-19T03:13:28.299051095Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:13:28.635564552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:13:30.185919311Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-19T03:13:30.185941686Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=9
level=warn ts=2026-05-19T03:13:33.635224918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:13:33.987595352Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-19T03:13:35.045376284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:13:35.053836506Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:13:37.790180608Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-19T03:13:38.635082099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:13:38.635110595Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:13:38.635134753Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:13:38.635145249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:13:39.255883208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:13:41.593647672Z caller=checkpoint.go:372 msg="writing series" size="381 kB"
level=warn ts=2026-05-19T03:13:43.635938344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:13:45.396430422Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-19T03:13:48.635612844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:13:48.635622511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47136"
level=debug ts=2026-05-19T03:13:49.199290531Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:13:49.199318826Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-05-19T03:13:53.001693474Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T03:13:53.635165392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:13:56.809723608Z caller=checkpoint.go:372 msg="writing series" size="8.9 MB"
level=debug ts=2026-05-19T03:13:56.80974591Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.9 MB" series=2
level=warn ts=2026-05-19T03:13:58.634921855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:13:59.32617867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:14:00.607130312Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-19T03:14:03.63538566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:14:04.410635147Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
ts=2026-05-19T03:14:07.059493803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39762"
level=debug ts=2026-05-19T03:14:07.529447052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:14:08.213733464Z caller=checkpoint.go:372 msg="writing series" size="451 kB"
level=info ts=2026-05-19T03:14:08.617305347Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:14:08.617760014Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T03:14:08.635424458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:14:08.635435198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:14:08.635450633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:14:08.635458874Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:14:09.257601097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:14:12.016454056Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-19T03:14:13.635723805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:14:14.640661689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:14:14.647468943Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:14:14.647491096Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:14:14.647517131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:14:14.653546288Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:14:14.951817605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:14:14.958075564Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:14:15.818609246Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-19T03:14:18.635404993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:14:18.638459873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53960"
level=debug ts=2026-05-19T03:14:19.621773099Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-19T03:14:23.424212433Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-19T03:14:23.424233774Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-19T03:14:23.635843758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:14:27.227374542Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-05-19T03:14:28.635710225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:14:31.029753867Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-19T03:14:31.720588891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:14:31.73091225Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:14:32.607190143Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:14:33.635124074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:14:34.83325874Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-19T03:14:38.635289279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:14:38.635314128Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:14:38.635310058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:14:38.635321862Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:14:38.635321425Z caller=checkpoint.go:384 msg="flushing series" totalSize="522 kB" series=3
level=info ts=2026-05-19T03:14:38.640610353Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-19T03:14:38.645333539Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.01022928s
ts=2026-05-19T03:14:39.258462338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=warn ts=2026-05-19T03:14:43.635736058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:14:43.657910498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:14:48.635383052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:14:53.634958654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:14:58.386162672Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:14:58.38622001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:14:58.393506345Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:14:58.635216986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:14:58.784518599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:14:58.790890002Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:15:03.635926003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:15:06.367586566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:15:06.377005628Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:15:08.617445636Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:15:08.617451788Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-19T03:15:08.618754Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-19T03:15:08.61879101Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976844 n=1
level=debug ts=2026-05-19T03:15:08.618831183Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976844 n=1
level=info ts=2026-05-19T03:15:08.63536637Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T03:15:08.635392507Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:15:08.635410262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:15:08.635417507Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:15:08.635424541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:15:08.635470157Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-19T03:15:08.635478639Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:15:08.635494105Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.197µs success=true
level=debug ts=2026-05-19T03:15:08.636007801Z caller=checkpoint.go:372 msg="writing series" size="754 kB"
ts=2026-05-19T03:15:09.260021834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:15:09.860188253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:15:09.865646165Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:15:12.438956106Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T03:15:13.635237536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:15:13.850173636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41802"
level=debug ts=2026-05-19T03:15:16.242088371Z caller=checkpoint.go:372 msg="writing series" size="958 kB"
level=debug ts=2026-05-19T03:15:16.242110712Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-05-19T03:15:18.635706501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:15:18.644972532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59398"
level=debug ts=2026-05-19T03:15:20.045167138Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T03:15:23.635718762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:15:23.847927943Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-19T03:15:27.650611483Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=warn ts=2026-05-19T03:15:28.635866152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:15:31.047798017Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:15:31.453149752Z caller=checkpoint.go:372 msg="writing series" size="634 kB"
level=warn ts=2026-05-19T03:15:33.635210159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:15:35.256052737Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=debug ts=2026-05-19T03:15:35.256076659Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-19T03:15:35.622308222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:15:35.623931026Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T03:15:38.635344235Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:15:38.635364208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:15:38.635372364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:15:38.635386022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:15:39.058250087Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
ts=2026-05-19T03:15:39.261323442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:15:39.698032078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:15:39.70411718Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:15:42.303134043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:15:42.30982266Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:15:42.861383913Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T03:15:43.63544794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:15:43.851750067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49346"
level=debug ts=2026-05-19T03:15:46.664694356Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-19T03:15:48.635654111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:15:50.46698997Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-19T03:15:53.635570242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:15:54.270738553Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=debug ts=2026-05-19T03:15:58.073128786Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-19T03:15:58.392892173Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:15:58.635427853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:15:58.775752694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:15:58.78287487Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:16:01.882751206Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:16:01.88277344Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=7
level=warn ts=2026-05-19T03:16:03.635281697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:16:05.678002811Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:16:08.617443388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:16:08.617951293Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T03:16:08.634946509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:16:08.634963652Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:16:08.634980258Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:16:08.634987736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:16:09.262314307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:16:09.481479134Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=debug ts=2026-05-19T03:16:13.284463147Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:16:13.635234403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:16:17.086956203Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=warn ts=2026-05-19T03:16:18.635641774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:16:20.521609559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:16:20.889574164Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-19T03:16:23.627283158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:16:23.63245559Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:16:23.635508403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:16:24.419087867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:16:24.692870769Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=debug ts=2026-05-19T03:16:28.495475161Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-19T03:16:28.635552287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:16:32.298656433Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=warn ts=2026-05-19T03:16:33.635828664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:16:34.148845972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:16:34.157065323Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:16:36.101160059Z caller=checkpoint.go:372 msg="writing series" size="510 B"
ts=2026-05-19T03:16:37.070221244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42004"
level=info ts=2026-05-19T03:16:38.635423602Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:16:38.635446267Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:16:38.635439994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:16:38.635455015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:16:39.263652044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:16:39.903798805Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-19T03:16:43.635797172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:16:43.706967959Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-19T03:16:43.887096324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:16:43.897771097Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:16:47.509260842Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-19T03:16:48.452936647Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:16:48.635145522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:16:48.655408217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60144"
level=debug ts=2026-05-19T03:16:51.312607674Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-19T03:16:51.312630944Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-19T03:16:53.635090681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:16:55.115384303Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=debug ts=2026-05-19T03:16:57.932151206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:16:58.635615813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:16:58.918247657Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=debug ts=2026-05-19T03:17:02.721280877Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-19T03:17:03.635281733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:17:06.523176512Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-19T03:17:08.617720194Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:17:08.618491153Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T03:17:08.635158613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:17:08.635162172Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:17:08.635178368Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:17:08.635186082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:17:09.264479119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:17:10.326907054Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=warn ts=2026-05-19T03:17:13.635870316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:17:14.12911585Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-19T03:17:17.932484278Z caller=checkpoint.go:372 msg="writing series" size="469 kB"
level=warn ts=2026-05-19T03:17:18.635531031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:17:18.658796045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58842"
level=debug ts=2026-05-19T03:17:19.497513574Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:17:21.735266721Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
level=debug ts=2026-05-19T03:17:21.735288066Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-05-19T03:17:23.58310829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:17:23.589334866Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:17:23.635915759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:17:25.537200633Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=info ts=2026-05-19T03:17:28.353492693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:17:28.360449109Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:17:28.635005227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:17:29.339950715Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-19T03:17:33.142853246Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=warn ts=2026-05-19T03:17:33.6357561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:17:36.946201095Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=info ts=2026-05-19T03:17:38.635471715Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:17:38.635499035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:17:38.635506943Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:17:38.635501555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:17:39.266077436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:17:40.623923447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:17:40.632366308Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:17:40.748602613Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T03:17:43.63591858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:17:43.797385072Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:17:44.552126953Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-19T03:17:48.355070859Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-19T03:17:48.355091882Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-19T03:17:48.635403596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:17:50.491439742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:17:52.157489045Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-19T03:17:53.635731108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:17:55.960242794Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-19T03:17:58.635354618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:17:59.763442094Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-19T03:18:03.565831611Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-19T03:18:03.635788609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:18:07.368614909Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-19T03:18:08.617742482Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:18:08.61848023Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:18:08.635507693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:18:08.63552877Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:18:08.635551539Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:18:08.635564075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:18:09.267310526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:18:09.757189808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:18:09.765271297Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:18:11.171102763Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-19T03:18:13.63509008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:18:14.974098095Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-05-19T03:18:15.958646805Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:18:18.635413618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:18:18.665056319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47882"
level=debug ts=2026-05-19T03:18:18.776604953Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-19T03:18:22.580135801Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-19T03:18:22.886153494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:18:22.893621824Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:18:23.635471883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:18:26.382654932Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=debug ts=2026-05-19T03:18:27.238688745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:18:28.635893125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:18:30.186719587Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-19T03:18:30.186741303Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=11
level=info ts=2026-05-19T03:18:32.204657587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:18:32.211082433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:18:33.635059681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:18:33.988587618Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-19T03:18:37.790877986Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-19T03:18:38.635427874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:18:38.635439498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:18:38.635462316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:18:38.635469742Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:18:39.268557243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:18:41.594606963Z caller=checkpoint.go:372 msg="writing series" size="392 kB"
level=warn ts=2026-05-19T03:18:43.635922645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:18:44.0696086Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:18:45.396885087Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-19T03:18:48.635096641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:18:49.200505581Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:18:49.200528368Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-05-19T03:18:53.002561419Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T03:18:53.635095797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:18:56.814443618Z caller=checkpoint.go:372 msg="writing series" size="9.9 MB"
level=debug ts=2026-05-19T03:18:56.814470963Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.9 MB" series=2
level=warn ts=2026-05-19T03:18:58.63543924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:19:00.608413586Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-19T03:19:03.635510096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:19:04.177412399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:19:04.184951377Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:19:04.410629497Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-19T03:19:04.882606571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:19:04.884368197Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:19:07.081175409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39830"
level=debug ts=2026-05-19T03:19:08.213680045Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=info ts=2026-05-19T03:19:08.617450315Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:19:08.617470528Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:19:08.635862392Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:19:08.635863272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:19:08.635885662Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:19:08.635895394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:19:09.269878861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:19:11.111448891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:19:11.11727634Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:19:12.016905083Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-19T03:19:13.635067603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:19:15.819624181Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-19T03:19:16.00400806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:19:18.635561083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:19:19.622745316Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=debug ts=2026-05-19T03:19:23.425343691Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-19T03:19:23.425369091Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-19T03:19:23.635154868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:19:27.227324199Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-19T03:19:27.858077986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:19:27.86713231Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:19:28.635387857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:19:31.03094008Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-19T03:19:33.635408446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:19:34.833376369Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-19T03:19:38.634834044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:19:38.634855445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:19:38.634863152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:19:38.634887367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:19:38.63588226Z caller=checkpoint.go:384 msg="flushing series" totalSize="522 kB" series=3
level=info ts=2026-05-19T03:19:38.641835134Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-19T03:19:38.644992327Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009366921s
ts=2026-05-19T03:19:39.271340263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=warn ts=2026-05-19T03:19:43.635367855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:19:43.867621599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45144"
level=info ts=2026-05-19T03:19:44.980146673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:19:44.988093997Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:19:46.588467214Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:19:48.635230397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:19:48.675006022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50822"
level=info ts=2026-05-19T03:19:52.961802318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:19:52.967503642Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:19:53.63491873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:19:58.635428686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:20:03.635226559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:20:05.551800276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:20:07.084053446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47766"
level=debug ts=2026-05-19T03:20:08.617198785Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:20:08.618254548Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T03:20:08.634678073Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T03:20:08.634796508Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-19T03:20:08.634804378Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:20:08.634820581Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.825µs success=true
level=warn ts=2026-05-19T03:20:08.634888692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:20:08.634890836Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:20:08.63490819Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:20:08.634915739Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:20:08.635454475Z caller=checkpoint.go:372 msg="writing series" size="864 kB"
ts=2026-05-19T03:20:09.2740965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:20:12.438600639Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T03:20:13.635875223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:20:13.868771863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48514"
level=debug ts=2026-05-19T03:20:16.241425209Z caller=checkpoint.go:372 msg="writing series" size="976 kB"
level=debug ts=2026-05-19T03:20:16.241447382Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=warn ts=2026-05-19T03:20:18.635053158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:20:19.248305232Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:20:20.04434386Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T03:20:23.635143699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:20:23.84713198Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T03:20:26.08447529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:20:26.092944248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:20:26.09297099Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:20:26.093008625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:20:26.099832408Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:20:27.174688699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:20:27.180920803Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:20:27.180940435Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:20:27.180963505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:20:27.186771204Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:20:27.64941615Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-19T03:20:28.635502923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:20:31.452286031Z caller=checkpoint.go:372 msg="writing series" size="479 kB"
level=warn ts=2026-05-19T03:20:33.635588957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:20:35.255246263Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=debug ts=2026-05-19T03:20:35.255269476Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-19T03:20:38.634921592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:20:38.634944862Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:20:38.634952394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:20:38.634969022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:20:39.058062977Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
ts=2026-05-19T03:20:39.277788355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:20:40.103471756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:20:40.11046713Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:20:42.177097204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:20:42.183947626Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:20:42.8610695Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T03:20:43.635321672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:20:44.677106506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:20:44.683461138Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:20:46.663741695Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-19T03:20:48.635905324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:20:50.466385343Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-19T03:20:53.635472889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:20:54.26915942Z caller=checkpoint.go:372 msg="writing series" size="349 kB"
level=debug ts=2026-05-19T03:20:54.778488932Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:20:58.072183067Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T03:20:58.635787743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:20:59.181723833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:21:01.883901259Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:21:01.88392721Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=7
level=info ts=2026-05-19T03:21:02.924132429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:21:02.930950167Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:21:03.635094871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:21:05.678053846Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:21:08.617457541Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:21:08.617505681Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:21:08.635650261Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:21:08.63566278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:21:08.63567246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:21:08.63568574Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:21:09.279779528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:21:09.481067439Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-19T03:21:13.283351857Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:21:13.635128243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:21:13.873130375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34408"
level=debug ts=2026-05-19T03:21:17.086138509Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=info ts=2026-05-19T03:21:18.419865715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:21:18.426796383Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:21:18.635201171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:21:20.889289748Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-19T03:21:23.635287515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:21:24.692375107Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=debug ts=2026-05-19T03:21:25.79777352Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:21:25.79782865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:21:25.801031487Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:21:28.494986124Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-19T03:21:28.63537583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:21:32.297171539Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-19T03:21:33.635102394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:21:36.100231935Z caller=checkpoint.go:372 msg="writing series" size="642 B"
level=info ts=2026-05-19T03:21:38.635818464Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:21:38.635842456Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:21:38.635850309Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:21:38.635843552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:21:39.273235681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-19T03:21:39.280498251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=error ts=2026-05-19T03:21:39.280693529Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:21:39.280712222Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:21:39.280735921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:21:39.285305797Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:21:39.903042381Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-19T03:21:42.796715202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:21:42.801770049Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:21:43.635849616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:21:43.706046572Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-19T03:21:46.35457661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:21:46.360687834Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:21:47.509050103Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-19T03:21:48.634975953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:21:51.249722869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:21:51.256703442Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:21:51.311985299Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-19T03:21:51.312002981Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-19T03:21:53.635107684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:21:55.114757216Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-19T03:21:58.635429688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:21:58.917765539Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-19T03:21:58.961261878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:22:01.687659339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:22:02.720214507Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-19T03:22:03.635196309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:22:06.52321725Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-19T03:22:07.096074752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37884"
level=info ts=2026-05-19T03:22:08.007060865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:22:08.010720105Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:22:08.617553359Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:22:08.618092693Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:22:08.634963612Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:22:08.634984898Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:22:08.634999584Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:22:08.634994707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:22:09.282060101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:22:10.325735485Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=info ts=2026-05-19T03:22:12.149732637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:22:12.155431359Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:22:13.635049983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:22:14.129011888Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T03:22:16.239459515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:22:16.249633961Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:22:17.931609508Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
level=warn ts=2026-05-19T03:22:18.6351735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:22:21.734562967Z caller=checkpoint.go:372 msg="writing series" size="470 kB"
level=debug ts=2026-05-19T03:22:21.734586643Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-19T03:22:23.63502212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:22:25.536699836Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=warn ts=2026-05-19T03:22:28.63570228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:22:29.339639514Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-19T03:22:29.516164157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:22:29.524441033Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:22:31.414874974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:22:33.142831106Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-19T03:22:33.635728561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:22:36.945181143Z caller=checkpoint.go:372 msg="writing series" size="562 kB"
level=debug ts=2026-05-19T03:22:36.945196821Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=4
ts=2026-05-19T03:22:37.09792407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:43768"
level=warn ts=2026-05-19T03:22:38.635476644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:22:38.635483479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:22:38.635502064Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:22:38.635510037Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:22:39.28386373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:22:40.480376375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:22:40.748319159Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T03:22:43.635490289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:22:44.551522542Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-19T03:22:48.353905362Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-19T03:22:48.635708881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:22:48.694800789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49298"
level=info ts=2026-05-19T03:22:49.33673734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:22:49.341135088Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:22:52.157112127Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-19T03:22:53.635487066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:22:55.959161774Z caller=checkpoint.go:372 msg="writing series" size="345 kB"
level=warn ts=2026-05-19T03:22:58.635325729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:22:59.762716486Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-19T03:22:59.801864053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:23:03.564621656Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-19T03:23:03.635613404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:23:06.155071121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:23:06.160973462Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:23:07.034398672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:23:07.041027544Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:23:07.367766536Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-05-19T03:23:08.617235735Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:23:08.618298742Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T03:23:08.635320703Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:23:08.635338177Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:23:08.63535095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:23:08.635354847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:23:09.285804381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:23:11.170547481Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-19T03:23:13.635455314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:23:14.973909259Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=warn ts=2026-05-19T03:23:18.634933094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:23:18.69795161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43364"
level=debug ts=2026-05-19T03:23:18.776515467Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-05-19T03:23:22.579289076Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-19T03:23:23.635563355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:23:25.032334631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:23:25.041498555Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:23:26.38214256Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-19T03:23:28.635573078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:23:30.186216234Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-19T03:23:30.186237152Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=14
level=debug ts=2026-05-19T03:23:33.088225393Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:23:33.635491954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:23:33.988164517Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-19T03:23:36.802028577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:23:36.803484448Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:23:37.102325655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40858"
level=debug ts=2026-05-19T03:23:37.790705906Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-19T03:23:38.634885728Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:23:38.634912152Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:23:38.63492Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:23:38.634920181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:23:39.28745311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:23:41.593597439Z caller=checkpoint.go:372 msg="writing series" size="397 kB"
level=info ts=2026-05-19T03:23:42.33446279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:23:42.337775001Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:23:43.635029026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:23:43.882107007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47976"
level=debug ts=2026-05-19T03:23:45.396368278Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-19T03:23:48.635841378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:23:49.199145415Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:23:49.19916672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-19T03:23:50.746302182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:23:50.753074577Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:23:53.001678766Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T03:23:53.635019168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:23:56.80805093Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-19T03:23:56.808073632Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=warn ts=2026-05-19T03:23:58.635581874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:24:00.212457933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:24:00.217607527Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:24:00.232729098Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:24:00.607862205Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=warn ts=2026-05-19T03:24:03.63563105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:24:04.409917013Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=debug ts=2026-05-19T03:24:08.213100787Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=info ts=2026-05-19T03:24:08.617313134Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:24:08.617793635Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T03:24:08.635312892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:24:08.635325293Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:24:08.635340107Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:24:08.635347494Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:24:09.288736008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:24:12.016338456Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-19T03:24:13.635416964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:24:13.88362739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49682"
level=debug ts=2026-05-19T03:24:15.819018837Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=debug ts=2026-05-19T03:24:15.819042903Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-19T03:24:18.635168589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:24:18.704546549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47650"
level=debug ts=2026-05-19T03:24:19.621862662Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=debug ts=2026-05-19T03:24:20.690008043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:24:23.323586757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:24:23.329514273Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:24:23.423895539Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-19T03:24:23.635404946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:24:24.451951743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:24:24.455040708Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:24:27.227425063Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-05-19T03:24:28.635574133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:24:29.350243593Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:24:31.030430436Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-19T03:24:31.077426315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:24:31.083797486Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:24:33.635431505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:24:34.833060724Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=debug ts=2026-05-19T03:24:38.635216409Z caller=checkpoint.go:384 msg="flushing series" totalSize="867 kB" series=5
level=info ts=2026-05-19T03:24:38.635221925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:24:38.635242313Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:24:38.635251199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:24:38.635271247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:24:38.642045779Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-19T03:24:38.645111458Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010176797s
ts=2026-05-19T03:24:39.289934263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=warn ts=2026-05-19T03:24:43.635601317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:24:48.635348634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:24:48.707523624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39896"
level=info ts=2026-05-19T03:24:48.903954545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:24:48.910098422Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:24:53.634930867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:24:57.256432917Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:24:58.635513341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:25:03.635476707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:25:07.333070219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:25:07.338493512Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:25:08.617462832Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:25:08.617736939Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:25:08.634765969Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T03:25:08.634897163Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-19T03:25:08.634907537Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:25:08.634924911Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.019µs success=true
level=info ts=2026-05-19T03:25:08.634983547Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:25:08.635003145Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:25:08.63501286Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:25:08.63504317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:25:08.635479987Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
ts=2026-05-19T03:25:09.291632195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:25:12.438972399Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-19T03:25:12.558652301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:25:13.634992176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:25:16.242085135Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-19T03:25:16.242104965Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-19T03:25:16.268780891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:25:16.274841135Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:25:18.613453318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:25:18.619824554Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:25:18.634905864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:25:18.711191864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52332"
level=debug ts=2026-05-19T03:25:20.043596492Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T03:25:23.635567245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:25:23.847321934Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-19T03:25:25.203670509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:25:27.649789363Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=warn ts=2026-05-19T03:25:28.635360941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:25:31.453304441Z caller=checkpoint.go:372 msg="writing series" size="737 kB"
level=warn ts=2026-05-19T03:25:33.635273565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:25:35.255529527Z caller=checkpoint.go:372 msg="writing series" size="714 kB"
level=debug ts=2026-05-19T03:25:35.255570745Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
ts=2026-05-19T03:25:37.109194967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34284"
level=info ts=2026-05-19T03:25:38.635652861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:25:38.635663291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:25:38.63567743Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:25:38.635692451Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:25:39.05850017Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-19T03:25:39.292084813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-19T03:25:39.293259853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=error ts=2026-05-19T03:25:39.300850516Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:25:42.861342442Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T03:25:43.635139753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:25:43.889119559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46912"
level=debug ts=2026-05-19T03:25:46.664334844Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-19T03:25:47.440510154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:25:47.443675754Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:25:48.635146834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:25:50.466933247Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-19T03:25:53.635447796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:25:54.269815714Z caller=checkpoint.go:372 msg="writing series" size="349 kB"
level=debug ts=2026-05-19T03:25:55.633573437Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:25:58.072106344Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T03:25:58.635590018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:26:01.880946863Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:26:01.880970546Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=7
level=info ts=2026-05-19T03:26:03.30081448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:26:03.309321742Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:26:03.635244401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:26:05.677767028Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:26:08.617301042Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:26:08.617664002Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:26:08.635224202Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:26:08.635229213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:26:08.63523967Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:26:08.635247447Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:26:09.294342522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:26:09.480670465Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=info ts=2026-05-19T03:26:10.43936745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:26:10.44580919Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:26:12.982930107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:26:13.283780913Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:26:13.635557771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:26:17.086886307Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=warn ts=2026-05-19T03:26:18.635073145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:26:18.717593426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42542"
level=debug ts=2026-05-19T03:26:20.889017003Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-19T03:26:23.635270191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:26:24.691530821Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=debug ts=2026-05-19T03:26:28.494387281Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-19T03:26:28.635932854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:26:30.817785188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:26:30.824711063Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:26:31.030353393Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:26:32.297631757Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T03:26:33.635504114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:26:36.100327778Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-19T03:26:36.336072242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:26:36.341118229Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:26:38.635812494Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:26:38.635839338Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:26:38.635848512Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:26:38.635866341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:26:39.295364881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:26:39.90322779Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-19T03:26:43.635638604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:26:43.705857551Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-19T03:26:47.176093064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:26:47.185801962Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:26:47.185823936Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:26:47.185851442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:26:47.191646825Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:26:47.50871249Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-19T03:26:48.635074779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:26:48.721653468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36976"
level=debug ts=2026-05-19T03:26:51.31169575Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-19T03:26:51.311717886Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-19T03:26:53.635730023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:26:55.106220231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:26:55.113606755Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:26:55.113626411Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:26:55.113652117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:26:55.114591062Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=error ts=2026-05-19T03:26:55.11986723Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:26:58.63510903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:26:58.917195733Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-19T03:26:59.862892942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:26:59.869959536Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:27:00.784660158Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:27:02.720760498Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-19T03:27:03.63577381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:27:06.522654302Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-19T03:27:08.6173112Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:27:08.61730682Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:27:08.634825502Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:27:08.634841947Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:27:08.634851478Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:27:08.634873791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:27:08.986295613Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:27:09.296739989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:27:10.326225695Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=info ts=2026-05-19T03:27:11.297736048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:27:11.304676972Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:27:13.635581816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:27:14.128739596Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T03:27:17.847271273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:27:17.855372125Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:27:17.931863049Z caller=checkpoint.go:372 msg="writing series" size="411 kB"
level=warn ts=2026-05-19T03:27:18.635425565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:27:21.734831998Z caller=checkpoint.go:372 msg="writing series" size="391 kB"
level=debug ts=2026-05-19T03:27:21.734856949Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-19T03:27:23.63558225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:27:24.575303003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:27:24.578709056Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:27:24.578727505Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:27:24.57874768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:27:24.581282155Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:27:25.537353118Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=debug ts=2026-05-19T03:27:26.226451059Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:27:28.635127282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:27:29.339690688Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-19T03:27:33.143224376Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
level=warn ts=2026-05-19T03:27:33.63518776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:27:35.138180067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:27:35.145962491Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:27:36.945708809Z caller=checkpoint.go:372 msg="writing series" size="561 kB"
level=debug ts=2026-05-19T03:27:36.945730487Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-19T03:27:38.478550274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:27:38.48487199Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:27:38.532206082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:27:38.535526208Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:27:38.635458668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:27:38.635482777Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:27:38.635492401Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:27:38.635495752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:27:39.29796881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:27:40.748049648Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T03:27:43.635608911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:27:43.899983181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56164"
level=debug ts=2026-05-19T03:27:44.32868455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:27:44.330213128Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:27:44.550850866Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-19T03:27:48.353877143Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-19T03:27:48.635508025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:27:52.156669282Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-19T03:27:53.63486467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:27:55.960035556Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-19T03:27:58.635432276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:27:59.762762848Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T03:28:00.246929474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:28:00.251476602Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:28:01.456966289Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:28:03.565664233Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-19T03:28:03.635782753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:28:05.19587106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:28:05.201197854Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:28:05.201217273Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:28:05.201239651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:28:05.205507993Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-19T03:28:07.117221722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33926"
level=debug ts=2026-05-19T03:28:07.368262123Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-05-19T03:28:08.617330403Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:28:08.617336112Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T03:28:08.634829839Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:28:08.634846442Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:28:08.634854345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:28:08.634876994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:28:09.299417181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:28:11.171227881Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-19T03:28:13.635758214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:28:13.901154252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42618"
level=debug ts=2026-05-19T03:28:14.973370703Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=warn ts=2026-05-19T03:28:18.635385021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:28:18.731984966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50080"
level=debug ts=2026-05-19T03:28:18.776549585Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-19T03:28:22.105930276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:28:22.115064822Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:28:22.579174582Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-19T03:28:22.835215634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:28:22.841296904Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:28:23.634874171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:28:26.382241205Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-19T03:28:28.634968251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:28:29.073295626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:28:30.185659132Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-19T03:28:30.185683615Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=14
level=info ts=2026-05-19T03:28:30.399147929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:28:30.405831155Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:28:33.635874797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:28:33.987262552Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-19T03:28:35.184056822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="693 kB" avg_comp="693 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:28:35.184100854Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-19T03:28:35.187717952Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:28:37.790833139Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-19T03:28:38.635510382Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:28:38.6355497Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:28:38.635557698Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:28:38.635590137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:28:39.300738847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:28:41.593098704Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=warn ts=2026-05-19T03:28:43.635531392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:28:45.187908414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:28:45.191164474Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:28:45.396838768Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-19T03:28:48.635851829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:28:49.19902135Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:28:49.199042918Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-19T03:28:49.225947526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:28:49.231326453Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:28:50.360209962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:28:50.364477741Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:28:53.001753391Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T03:28:53.635317932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:28:56.814082114Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-19T03:28:56.814102627Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=warn ts=2026-05-19T03:28:58.634964582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:29:00.608072599Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-19T03:29:03.63503439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:29:04.410845558Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
ts=2026-05-19T03:29:07.121746564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:57526"
level=debug ts=2026-05-19T03:29:07.747326775Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:29:08.21305761Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-19T03:29:08.617801696Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:29:08.618664014Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:29:08.635121853Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:29:08.635140105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:29:08.63514914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:29:08.635165357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:29:09.303105806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:29:12.016440083Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-19T03:29:12.316097315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:29:12.662090797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:29:12.668370218Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:29:13.635270188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:29:13.775581853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:29:13.7791012Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:29:15.81878172Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-19T03:29:18.572644798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:29:18.579122022Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:29:18.635848497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:29:18.73966759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44792"
level=debug ts=2026-05-19T03:29:19.621523403Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-05-19T03:29:19.621568663Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-19T03:29:23.424141604Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-19T03:29:23.635628886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:29:27.226928682Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-05-19T03:29:28.635791349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:29:31.03062702Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-19T03:29:33.635200989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:29:34.833095233Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-19T03:29:38.102062298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:29:38.106554016Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:29:38.635716335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:29:38.635727019Z caller=checkpoint.go:384 msg="flushing series" totalSize="661 kB" series=4
level=debug ts=2026-05-19T03:29:38.635741205Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:29:38.635749488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:29:38.635747214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:29:38.641107838Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-19T03:29:38.64423023Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009151314s
ts=2026-05-19T03:29:39.306754251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:29:39.4237128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:29:39.428591718Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:29:42.893664419Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:29:43.635282624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:29:48.635838315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:29:51.635014021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:29:53.634882198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:29:57.87695302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:29:57.883977489Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:29:58.634884639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:30:03.635579204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:30:05.195687246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:30:05.200080629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:30:08.617418872Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:30:08.617830641Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-19T03:30:08.619086738Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-19T03:30:08.619124971Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976845 n=1
level=debug ts=2026-05-19T03:30:08.619162676Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976845 n=1
level=info ts=2026-05-19T03:30:08.635404292Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-19T03:30:08.635413392Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-19T03:30:08.635425104Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:30:08.635435159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:30:08.63543782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:30:08.635564226Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-19T03:30:08.635573105Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:30:08.635594179Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.92µs success=true
level=debug ts=2026-05-19T03:30:08.636090325Z caller=checkpoint.go:372 msg="writing series" size="750 kB"
level=debug ts=2026-05-19T03:30:09.085072532Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T03:30:09.308884181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:30:12.439402566Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T03:30:13.63547548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:30:13.909014591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48672"
level=debug ts=2026-05-19T03:30:16.242054498Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-19T03:30:16.242076271Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-19T03:30:18.52979701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:30:18.537695307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:30:18.635468524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:30:18.745883929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37242"
level=debug ts=2026-05-19T03:30:20.0452687Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T03:30:23.634914197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:30:23.848097264Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T03:30:26.272051976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:30:26.279479068Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:30:27.650331065Z caller=checkpoint.go:372 msg="writing series" size="321 kB"
level=warn ts=2026-05-19T03:30:28.635642237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:30:31.453591868Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
level=debug ts=2026-05-19T03:30:31.646161814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:30:33.635413591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:30:35.256437745Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-19T03:30:35.256458839Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-19T03:30:36.412138869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:30:36.415253589Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-19T03:30:37.13045599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47384"
level=debug ts=2026-05-19T03:30:37.944153039Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:30:38.63498337Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:30:38.635004115Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:30:38.635014368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:30:38.635069387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:30:39.058904248Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
ts=2026-05-19T03:30:39.310647318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:30:42.861589749Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T03:30:43.635579738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:30:43.685689016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:30:43.692341797Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-19T03:30:43.910704136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51822"
level=debug ts=2026-05-19T03:30:46.664450092Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-19T03:30:48.635623083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:30:50.466933041Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-19T03:30:53.635335119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:30:54.270632868Z caller=checkpoint.go:372 msg="writing series" size="360 kB"
level=debug ts=2026-05-19T03:30:58.072964839Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T03:30:58.634875236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:31:01.432681819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:31:01.438678254Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:31:01.881352216Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:31:01.881375738Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=7
level=warn ts=2026-05-19T03:31:03.635876402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:31:04.433380924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:31:04.440168555Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:31:05.678195896Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:31:08.617441724Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:31:08.617924708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:31:08.635568406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:31:08.635587154Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:31:08.635597478Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:31:08.635610027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:31:09.042219578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T03:31:09.311553699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:31:09.481585086Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-05-19T03:31:13.28412576Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:31:13.635901028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:31:13.911952131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47486"
level=debug ts=2026-05-19T03:31:17.087410108Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=info ts=2026-05-19T03:31:18.585585467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:31:18.591420257Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:31:18.63544189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:31:18.752932259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59180"
level=info ts=2026-05-19T03:31:20.862825063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:31:20.865920527Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:31:20.889993419Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-19T03:31:23.634885283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:31:24.693110936Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-19T03:31:28.495600818Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=warn ts=2026-05-19T03:31:28.635471176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:31:29.173139031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:31:29.179148075Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:31:30.845990981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:31:30.847686751Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:31:32.297779742Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-19T03:31:33.05315067Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:31:33.635707763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:31:36.101145446Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-19T03:31:38.635057032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:31:38.635070033Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:31:38.635088878Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:31:38.635097084Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:31:39.313021577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:31:39.904163046Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-19T03:31:43.635746998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:31:43.70697778Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-19T03:31:43.912649745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55408"
level=debug ts=2026-05-19T03:31:47.510011897Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-19T03:31:48.635428977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:31:50.187969036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:31:50.194851973Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:31:51.037484117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:31:51.044568785Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:31:51.312310377Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-19T03:31:51.312332739Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=13
level=warn ts=2026-05-19T03:31:53.635579014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:31:55.115195361Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=debug ts=2026-05-19T03:31:57.106650121Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:31:58.634893529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:31:58.918181275Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-19T03:32:00.432600819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:32:00.437743588Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:32:02.721189875Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-19T03:32:03.635036795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:32:06.524060534Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-19T03:32:07.137435102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39980"
level=info ts=2026-05-19T03:32:08.617409626Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:32:08.61746357Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T03:32:08.635564253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:32:08.635577355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:32:08.635597455Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:32:08.635604811Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:32:09.314589413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:32:10.326233647Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-19T03:32:13.635550602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:32:13.914078149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55288"
level=debug ts=2026-05-19T03:32:14.129256177Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T03:32:17.739487379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:32:17.743205539Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:32:17.932044728Z caller=checkpoint.go:372 msg="writing series" size="491 kB"
level=warn ts=2026-05-19T03:32:18.635790199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:32:18.759264218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49966"
level=debug ts=2026-05-19T03:32:21.734832752Z caller=checkpoint.go:372 msg="writing series" size="526 kB"
level=debug ts=2026-05-19T03:32:21.734855348Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-19T03:32:23.635364453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:32:24.017683965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:32:24.301108509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:32:24.307196802Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:32:25.379147932Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:32:25.537865663Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=warn ts=2026-05-19T03:32:28.634866427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:32:29.340564708Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-19T03:32:33.143767363Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-19T03:32:33.258144662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:32:33.261928534Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T03:32:33.635366292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:32:36.946874794Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
ts=2026-05-19T03:32:37.138589881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:36566"
level=info ts=2026-05-19T03:32:37.855019163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:32:37.861315607Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:32:38.635822093Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:32:38.635845129Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:32:38.63583388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:32:38.635852951Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:32:39.317108994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:32:40.749392605Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T03:32:43.63552757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:32:44.551528088Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-19T03:32:44.852565242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:32:44.861211529Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:32:48.355050214Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-19T03:32:48.35507375Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-19T03:32:48.635784766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:32:52.157478107Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-19T03:32:53.077148987Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:32:53.63577777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:32:55.960869362Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=warn ts=2026-05-19T03:32:58.634994522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:32:59.763173464Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-19T03:33:03.565563674Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-19T03:33:03.635098846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:33:05.958458596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:33:05.965953737Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:33:05.96597344Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:33:05.965999122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:33:05.971652169Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-19T03:33:07.140106103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:38904"
level=debug ts=2026-05-19T03:33:07.368998902Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-19T03:33:08.617126229Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:33:08.618168377Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-19T03:33:08.634886862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:33:08.634895719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:33:08.634913735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:33:08.634922527Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:33:09.319034537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:33:11.17130474Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-19T03:33:12.764463124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:33:13.635394624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:33:14.974747242Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-19T03:33:15.908369568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:33:15.913871043Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:33:15.959002728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:33:15.961598691Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:33:16.034947291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:33:16.040559407Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:33:18.635782558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:33:18.777119768Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=debug ts=2026-05-19T03:33:22.579528918Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-19T03:33:23.635080113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:33:26.043603369Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:33:26.383411968Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-19T03:33:28.635530489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:33:30.186742327Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-19T03:33:30.186763692Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=11
level=info ts=2026-05-19T03:33:31.580700869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:33:31.586840444Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:33:32.517521649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:33:32.523890725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:33:32.523909922Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:33:32.52393253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:33:32.529721177Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:33:33.635766605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:33:33.988526124Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-19T03:33:37.791707663Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-19T03:33:38.342293098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:33:38.34897851Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:33:38.635351777Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:33:38.635376532Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:33:38.635385916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:33:38.635391436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:33:39.319893828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:33:41.593789079Z caller=checkpoint.go:372 msg="writing series" size="425 kB"
level=warn ts=2026-05-19T03:33:43.635008239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:33:45.397126916Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=info ts=2026-05-19T03:33:45.964588197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:33:45.973243339Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:33:48.634906693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:33:48.768641714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52882"
level=debug ts=2026-05-19T03:33:49.200459301Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:33:49.200479982Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=debug ts=2026-05-19T03:33:50.286823852Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:33:53.003020986Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T03:33:53.635815774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:33:56.811824232Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-19T03:33:56.811846555Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=warn ts=2026-05-19T03:33:58.635389954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:33:59.179269629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:33:59.188981562Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:34:00.607873702Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-19T03:34:03.635349714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:34:04.410869267Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
ts=2026-05-19T03:34:07.142208059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:43390"
level=info ts=2026-05-19T03:34:08.174151501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:34:08.177680691Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:34:08.177701691Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:34:08.177723453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:34:08.180411744Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:34:08.213910725Z caller=checkpoint.go:372 msg="writing series" size="328 kB"
level=debug ts=2026-05-19T03:34:08.617200573Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:34:08.618242958Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-19T03:34:08.635874083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:34:08.635880267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:34:08.635894297Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:34:08.63590187Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:34:09.320797991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:34:12.016299924Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-19T03:34:12.450482994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:34:13.434404896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:34:13.441686226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:34:13.635764324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:34:15.542453076Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:34:15.819491128Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=info ts=2026-05-19T03:34:18.31120898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:34:18.315996963Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:34:18.316017307Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:34:18.316040893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:34:18.320474843Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:34:18.635363707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:34:19.62278182Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-05-19T03:34:19.622802762Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-19T03:34:22.63391142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:34:22.637270319Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:34:23.425342094Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-19T03:34:23.635023431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:34:27.228217563Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-19T03:34:28.552839082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:34:28.560355424Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:34:28.635352574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:34:31.030497911Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-19T03:34:33.635585069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:34:34.833013477Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-19T03:34:35.629261114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:34:35.635580901Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:34:38.635648646Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:34:38.63566911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:34:38.635662093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:34:38.635675895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:34:38.636731778Z caller=checkpoint.go:384 msg="flushing series" totalSize="661 kB" series=4
level=info ts=2026-05-19T03:34:38.642802673Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-19T03:34:38.646041771Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010321985s
ts=2026-05-19T03:34:39.323053916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=warn ts=2026-05-19T03:34:43.635105688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:34:43.921430696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45216"
level=debug ts=2026-05-19T03:34:44.611061458Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:34:47.154579962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:34:47.15892906Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:34:48.635606883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:34:53.635647939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:34:54.442409703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:34:54.446589696Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:34:54.446610732Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:34:54.446630874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:34:54.449954949Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:34:56.86106465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:34:56.866280536Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:34:58.634947255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:35:03.635447794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:35:05.639709503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:35:05.647719104Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:35:07.322212245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:35:07.325660047Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:35:08.61737063Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:35:08.617740415Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:35:08.635525317Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T03:35:08.63557349Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:35:08.635589921Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:35:08.635585757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:35:08.635597635Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T03:35:08.635645679Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-19T03:35:08.635654041Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:35:08.635669093Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.807µs success=true
level=debug ts=2026-05-19T03:35:08.636271676Z caller=checkpoint.go:372 msg="writing series" size="861 kB"
ts=2026-05-19T03:35:09.325507734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:35:10.011133696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:35:12.439442905Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-19T03:35:13.635493453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:35:15.705213921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:35:15.711090452Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:35:16.241873223Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-19T03:35:16.241896709Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=debug ts=2026-05-19T03:35:18.632885331Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:35:18.63497158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:35:20.044849468Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T03:35:23.635908134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:35:23.847498192Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-19T03:35:27.650723386Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
level=warn ts=2026-05-19T03:35:28.635327313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:35:30.481481161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:35:30.489166213Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:35:31.453185458Z caller=checkpoint.go:372 msg="writing series" size="620 kB"
level=warn ts=2026-05-19T03:35:33.634996222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:35:33.69716105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:35:33.701118346Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:35:35.256212656Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-19T03:35:35.256237099Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-19T03:35:38.635781257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:35:38.635807012Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:35:38.635816886Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:35:38.635818794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:35:39.059069076Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
ts=2026-05-19T03:35:39.326683817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:35:42.86207069Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T03:35:43.634884723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:35:43.925912744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41510"
level=debug ts=2026-05-19T03:35:46.664449986Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-05-19T03:35:47.381237876Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:35:48.635604692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:35:50.467113535Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-19T03:35:53.433100736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:35:53.439015031Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:35:53.635886969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:35:54.270401433Z caller=checkpoint.go:372 msg="writing series" size="361 kB"
level=debug ts=2026-05-19T03:35:58.073268925Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T03:35:58.635874075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:36:01.671213175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:36:01.678068098Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:36:01.832252153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:36:01.833776855Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:36:01.883992731Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:36:01.88401212Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=7
level=warn ts=2026-05-19T03:36:03.635576064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:36:04.895574084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:36:04.902111178Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:36:05.678358375Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:36:08.617622389Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:36:08.618039596Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T03:36:08.635075574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:36:08.635087632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:36:08.635104262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:36:08.635111864Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:36:09.32746828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:36:09.481509504Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-19T03:36:13.284133465Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:36:13.635482044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:36:16.136994834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:36:16.144646358Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:36:17.086718372Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=debug ts=2026-05-19T03:36:17.905009804Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:36:18.635429026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:36:20.890025324Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-19T03:36:23.635801234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:36:24.23135015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:36:24.234843703Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:36:24.693142168Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-19T03:36:28.495872975Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=warn ts=2026-05-19T03:36:28.63511267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:36:32.298348818Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-19T03:36:33.635059736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:36:36.100870954Z caller=checkpoint.go:372 msg="writing series" size="657 B"
ts=2026-05-19T03:36:37.154934976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:56622"
level=info ts=2026-05-19T03:36:38.635013947Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:36:38.635037132Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:36:38.635045477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:36:38.635084692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:36:39.329277898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:36:39.903565995Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-19T03:36:43.635114421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:36:43.707315122Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-19T03:36:45.413993073Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:36:47.509259052Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=info ts=2026-05-19T03:36:47.932508705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:36:47.939584616Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:36:48.635616195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:36:48.788942449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33256"
level=debug ts=2026-05-19T03:36:50.226157702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:36:51.312309459Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-19T03:36:51.312330664Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=13
level=info ts=2026-05-19T03:36:53.294115399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:36:53.299217803Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:36:53.635499398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:36:55.115401456Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-19T03:36:58.635200199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:36:58.917976646Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-19T03:37:00.305311425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:37:00.311848975Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:37:02.720735908Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-19T03:37:03.635707736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:37:04.480798305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:37:04.483984513Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:37:05.715189538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:37:05.718865689Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:37:06.524077577Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
ts=2026-05-19T03:37:07.156253677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60970"
level=info ts=2026-05-19T03:37:08.617321577Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:37:08.617621703Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:37:08.635091906Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:37:08.635112992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:37:08.63512431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:37:08.63513326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:37:09.331982307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:37:10.326236977Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=warn ts=2026-05-19T03:37:13.635518394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:37:13.713890966Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T03:37:13.92990112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49758"
level=debug ts=2026-05-19T03:37:14.129244915Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-19T03:37:17.932807057Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
level=warn ts=2026-05-19T03:37:18.63564614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:37:21.735065473Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
level=debug ts=2026-05-19T03:37:21.735092939Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-19T03:37:23.635327448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:37:25.537676679Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-19T03:37:28.635880563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:37:29.341109823Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=debug ts=2026-05-19T03:37:33.143546949Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=warn ts=2026-05-19T03:37:33.635296969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:37:36.721915428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:37:36.727846505Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:37:36.946684293Z caller=checkpoint.go:372 msg="writing series" size="564 kB"
ts=2026-05-19T03:37:37.157034081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:50064"
level=info ts=2026-05-19T03:37:38.635566225Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:37:38.635572506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:37:38.635589091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:37:38.635596621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:37:39.332902746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:37:40.083288464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:37:40.089667103Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:37:40.748858585Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-19T03:37:43.635580478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:37:44.55240445Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-19T03:37:45.55825379Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:37:48.354619654Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-19T03:37:48.354641657Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-19T03:37:48.635790815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:37:48.759116125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:37:48.766068975Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:37:51.795648431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:37:51.802195164Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:37:52.157229821Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-19T03:37:52.417337993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:37:53.635288107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:37:55.960085035Z caller=checkpoint.go:372 msg="writing series" size="318 kB"
level=warn ts=2026-05-19T03:37:58.635387391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:37:59.763686177Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T03:38:03.28468599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:38:03.290164222Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:38:03.565915065Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-19T03:38:03.635869434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:38:07.368478055Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-19T03:38:08.617372701Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:38:08.617427321Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:38:08.635358336Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:38:08.635374408Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:38:08.635386513Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:38:08.635391875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:38:09.333813454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:38:10.752503625Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:38:11.171600007Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-19T03:38:13.635565336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:38:13.933862894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35180"
level=debug ts=2026-05-19T03:38:14.975083461Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-19T03:38:18.635677118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:38:18.776997309Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
ts=2026-05-19T03:38:18.798248218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40030"
level=debug ts=2026-05-19T03:38:22.580548192Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-19T03:38:23.635217353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:38:26.382744529Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=warn ts=2026-05-19T03:38:28.635576753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:38:30.186490527Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-19T03:38:30.186513463Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=11
level=info ts=2026-05-19T03:38:31.188295986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:38:31.196570416Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:38:33.634874314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:38:33.854322907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:38:33.859335203Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:38:33.988877697Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-19T03:38:35.494974002Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:38:37.790957105Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-19T03:38:38.635001903Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:38:38.635008665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:38:38.635025995Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:38:38.635034045Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:38:39.336228306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:38:41.594434317Z caller=checkpoint.go:372 msg="writing series" size="419 kB"
level=debug ts=2026-05-19T03:38:43.230273762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:38:43.635068851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:38:43.934925938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36634"
level=debug ts=2026-05-19T03:38:45.396902397Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-05-19T03:38:45.550388352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:38:45.554500736Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:38:48.635619244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:38:49.200013356Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:38:49.200036807Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-19T03:38:50.748256049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:38:50.758646305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:38:53.002902863Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-19T03:38:53.635204639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:38:56.813711203Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-19T03:38:56.813733865Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=warn ts=2026-05-19T03:38:58.635391025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:38:59.851117154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:38:59.854898336Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:39:00.608282093Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=warn ts=2026-05-19T03:39:03.635189306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:39:04.410821021Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-19T03:39:07.791315774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:39:08.213754978Z caller=checkpoint.go:372 msg="writing series" size="483 kB"
level=info ts=2026-05-19T03:39:08.617306429Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:39:08.617592107Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:39:08.634825947Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:39:08.63484011Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:39:08.634847821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:39:08.634860805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:39:09.33962574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:39:12.016552584Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-19T03:39:13.635583346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:39:13.936597091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33490"
level=debug ts=2026-05-19T03:39:15.81981058Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-19T03:39:17.688723624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:39:17.69480377Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:39:18.635531518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:39:19.622648835Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=debug ts=2026-05-19T03:39:19.622669672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-05-19T03:39:23.425118373Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-19T03:39:23.635882415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:39:27.228412827Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-05-19T03:39:28.634922724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:39:28.84743308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:39:28.858847559Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:39:28.85886913Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:39:28.858896257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:39:28.865153578Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:39:31.030728117Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-19T03:39:33.63584911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:39:34.833606124Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=debug ts=2026-05-19T03:39:36.270048868Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T03:39:37.167022007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47016"
level=info ts=2026-05-19T03:39:38.63509035Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:39:38.635113853Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:39:38.635121949Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:39:38.635129856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:39:38.636202647Z caller=checkpoint.go:384 msg="flushing series" totalSize="661 kB" series=4
level=info ts=2026-05-19T03:39:38.6427889Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-19T03:39:38.64608662Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010289694s
ts=2026-05-19T03:39:39.343519206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:39:41.801949413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:39:41.809785392Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:39:42.702426958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:39:43.635518908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:39:43.937674992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41474"
level=info ts=2026-05-19T03:39:45.324785557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:39:45.32786342Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:39:46.757576786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:39:46.763626224Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:39:48.635653728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:39:48.808122494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:32822"
level=info ts=2026-05-19T03:39:48.945667825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:39:48.949104927Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T03:39:53.635265021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:39:58.635210983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T03:40:03.635558491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:40:07.594730476Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:40:08.61741177Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:40:08.617926421Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:40:08.635055343Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T03:40:08.635080851Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:40:08.635098031Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:40:08.635106262Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:40:08.635117909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:40:08.635193259Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-19T03:40:08.635202728Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:40:08.635221615Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.832µs success=true
level=debug ts=2026-05-19T03:40:08.636131883Z caller=checkpoint.go:372 msg="writing series" size="953 kB"
level=info ts=2026-05-19T03:40:08.834067609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:40:08.839672599Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-19T03:40:09.34586166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:40:12.438593864Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-19T03:40:12.928013074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:40:12.934883631Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:40:13.635395419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:40:13.939042212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43588"
level=debug ts=2026-05-19T03:40:16.241853471Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-19T03:40:16.241875638Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=warn ts=2026-05-19T03:40:18.635092365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:40:18.811558738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43594"
level=debug ts=2026-05-19T03:40:19.170214685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:40:19.171915844Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:40:20.044353933Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-19T03:40:23.635045364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:40:23.846870039Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T03:40:26.599561918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:40:26.603507178Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:40:26.603526579Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:40:26.603570467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:40:26.606287977Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:40:27.650592624Z caller=checkpoint.go:372 msg="writing series" size="392 kB"
level=warn ts=2026-05-19T03:40:28.635620612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:40:30.509139191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:40:30.514135641Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:40:31.453297998Z caller=checkpoint.go:372 msg="writing series" size="670 kB"
level=debug ts=2026-05-19T03:40:31.453335678Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-19T03:40:33.635389502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:40:35.256036615Z caller=checkpoint.go:372 msg="writing series" size="646 kB"
ts=2026-05-19T03:40:37.170263212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55120"
level=info ts=2026-05-19T03:40:37.274880316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:40:37.281965449Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:40:37.281984269Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:40:37.28200988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:40:37.288187627Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:40:38.634922311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:40:38.634937678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:40:38.634959109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:40:38.634969155Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:40:39.058268824Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
ts=2026-05-19T03:40:39.348436846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:40:41.124200723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:40:41.127630566Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:40:42.509265258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:40:42.86132881Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-19T03:40:43.635291559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:40:46.663921939Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-19T03:40:48.635052026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:40:49.312134077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:40:49.318610529Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:40:50.466601332Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-19T03:40:53.635516922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:40:54.270127197Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=debug ts=2026-05-19T03:40:54.270148294Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-19T03:40:58.072328979Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-19T03:40:58.635120633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:40:59.887268099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:40:59.894226374Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:40:59.894246402Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:40:59.894268012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:40:59.898823348Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:41:01.883935042Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:41:01.883956669Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=warn ts=2026-05-19T03:41:03.634902636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:41:04.354459182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:41:05.678114428Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-05-19T03:41:07.171572314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45418"
level=info ts=2026-05-19T03:41:08.051470941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:41:08.057474972Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:41:08.617419889Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:41:08.617436715Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:41:08.635704326Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:41:08.635710489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:41:08.635721444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:41:08.635728946Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:41:09.350894448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:41:09.481004073Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=debug ts=2026-05-19T03:41:13.283609548Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:41:13.635371502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:41:13.944043279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45794"
level=info ts=2026-05-19T03:41:14.15497855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:41:14.158249753Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:41:14.300507556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:41:14.30511205Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:41:15.060319396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:41:15.063712646Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:41:15.063731076Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:41:15.06375234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:41:15.066681242Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:41:15.115891127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:41:15.121552622Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:41:16.884665903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:41:16.884680054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-19T03:41:16.887331595Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:41:17.087109994Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=info ts=2026-05-19T03:41:17.185181288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:41:17.18783593Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:41:17.484691589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="51 kB" avg_uncomp="51 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:41:17.487519107Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-19T03:41:17.785392114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="853 B" avg_uncomp="853 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:41:17.788899748Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:41:18.08564881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="30 kB" avg_uncomp="30 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:41:18.088370253Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:41:18.385142629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="138 kB" avg_uncomp="138 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:41:18.388006437Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-19T03:41:18.635086928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:41:18.685247062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="101 B" avg_uncomp="101 B" idle=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:41:18.687831983Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:41:20.889855731Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-19T03:41:23.635149981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:41:24.692118374Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T03:41:27.166647695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:41:27.171169688Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:41:28.494966451Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=warn ts=2026-05-19T03:41:28.635832842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:41:28.667901817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="51 kB" avg_uncomp="51 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:41:28.670916753Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-19T03:41:28.694977323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:41:28.697166424Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:41:29.896087633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:41:29.898690486Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:41:31.087827937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:41:31.090557215Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T03:41:32.297843877Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-19T03:41:33.635686192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:41:33.837751987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:41:33.840111559Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:41:35.769213016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:41:35.771799593Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:41:36.100817579Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-19T03:41:37.624511557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:41:37.627172467Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:41:38.635818227Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:41:38.635842779Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:41:38.635851448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:41:38.63584183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:41:39.353335112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:41:39.903241036Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T03:41:41.708409477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:41:41.713866155Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:41:43.061602979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=debug ts=2026-05-19T03:41:43.061669469Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-19T03:41:43.064301784Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:41:43.361957415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:41:43.366137427Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-19T03:41:43.635583376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:41:43.6616349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:41:43.664139525Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T03:41:43.706230986Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-19T03:41:43.945745383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43592"
level=info ts=2026-05-19T03:41:43.962293909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:41:43.964853248Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:41:44.262368229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:41:44.264928538Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:41:44.562589161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:41:44.565146958Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T03:41:47.509059155Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-19T03:41:48.635383947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:41:50.070560043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="51 kB" avg_uncomp="51 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:41:50.073230397Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-19T03:41:51.131801594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:41:51.135420122Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:41:51.312031696Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-19T03:41:51.312051566Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=warn ts=2026-05-19T03:41:53.635570063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:41:54.987580257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:41:54.990175006Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T03:41:55.114674917Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-19T03:41:56.312443758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:41:57.725578562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:41:57.727976789Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T03:41:58.635182258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:41:58.918096544Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-19T03:41:59.724196638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:41:59.726808638Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T03:42:02.083579899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:42:02.086158263Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:42:02.13336295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:42:02.137331187Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:42:02.720296269Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T03:42:02.741258943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:42:02.743869617Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:42:02.754911551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:42:02.756957929Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:42:03.07782321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:42:03.080528911Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T03:42:03.635481607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:42:04.283881775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:42:04.291676013Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:42:05.407598069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:42:05.409804726Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T03:42:06.523224748Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
ts=2026-05-19T03:42:07.175849392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:35060"
level=info ts=2026-05-19T03:42:07.442252659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:42:07.44869492Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:42:07.701393224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:42:07.703698506Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:42:08.617288237Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:42:08.617665971Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:42:08.635074691Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:42:08.635095524Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:42:08.63510345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:42:08.635161793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:42:09.161001675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:42:09.163998037Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-19T03:42:09.356846642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:42:09.607960823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:42:09.610447712Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:42:10.325814928Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T03:42:13.335269003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:42:13.338109813Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-19T03:42:13.635577776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:42:14.128521931Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-19T03:42:16.666116126Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:42:17.931642844Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-19T03:42:18.635728951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:42:18.824310942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48754"
level=info ts=2026-05-19T03:42:20.209636899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:42:20.212736514Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T03:42:21.734971052Z caller=checkpoint.go:372 msg="writing series" size="543 kB"
level=debug ts=2026-05-19T03:42:21.734995998Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-05-19T03:42:23.084079957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:42:23.089074031Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:42:23.635329994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:42:25.536937574Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-19T03:42:28.635688261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:42:29.339975398Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=info ts=2026-05-19T03:42:30.04707427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:42:30.049970296Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T03:42:33.142799135Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=warn ts=2026-05-19T03:42:33.635803092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:42:34.288367067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:42:34.294333567Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T03:42:34.811917577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:42:34.814726402Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:42:35.485798825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="51 kB" avg_uncomp="51 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:42:35.488482271Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-19T03:42:36.946114959Z caller=checkpoint.go:372 msg="writing series" size="566 kB"
ts=2026-05-19T03:42:37.17705216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60244"
level=info ts=2026-05-19T03:42:38.634830717Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:42:38.634853719Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:42:38.634862737Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:42:38.634889254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:42:38.954720816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:42:38.957087107Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-19T03:42:39.358732928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:42:40.619339546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:42:40.621959239Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T03:42:40.748219915Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-19T03:42:41.642955527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:42:41.645445908Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T03:42:43.635768089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:42:44.551587272Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-19T03:42:44.559785682Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:42:46.380473642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:42:46.383064316Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:42:47.229587355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:42:47.235824647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:42:48.353846137Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-19T03:42:48.353868835Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-19T03:42:48.635398911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:42:48.647459145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:42:48.653275292Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-19T03:42:48.827741317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52872"
level=info ts=2026-05-19T03:42:49.569741357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:42:49.573068592Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:42:52.15726966Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-19T03:42:52.565940754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:42:52.620271431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:42:52.622633989Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-19T03:42:53.635692172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:42:55.960050668Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-05-19T03:42:56.433418938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:42:56.43603753Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T03:42:58.635443124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:42:59.049412183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:42:59.051967593Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:42:59.762799011Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T03:43:03.298737959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:43:03.30168103Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:43:03.565433207Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-19T03:43:03.635467985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:43:06.078408171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:43:06.083989899Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:43:06.307589887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:43:06.310965016Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:43:07.368615621Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-19T03:43:08.617364659Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:43:08.61738964Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:43:08.634898377Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:43:08.634914961Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:43:08.634923009Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:43:08.634934236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:43:08.696474049Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T03:43:09.360394284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:43:10.874241132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:43:10.877242589Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T03:43:11.171024095Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-19T03:43:13.635443099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:43:14.917114447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:43:14.91976517Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T03:43:14.973899305Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-19T03:43:17.549913119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="51 kB" avg_uncomp="51 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:43:17.552978976Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-19T03:43:17.935578783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:43:17.939445889Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-19T03:43:18.635510725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:43:18.776835375Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=debug ts=2026-05-19T03:43:22.579594561Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-19T03:43:23.635366709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:43:24.760922117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:43:24.764038561Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T03:43:26.382134502Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=info ts=2026-05-19T03:43:27.756453491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:43:27.759341056Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T03:43:28.635045485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:43:29.301177265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:43:29.30777496Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:43:30.130111447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:43:30.134035714Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T03:43:30.186033007Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-19T03:43:30.186046222Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=11
level=info ts=2026-05-19T03:43:32.736445672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:43:32.738942336Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T03:43:33.635426581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:43:33.988319933Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T03:43:34.272122901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:43:34.274426725Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:43:34.984238464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:43:34.98845649Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:43:37.791219441Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=info ts=2026-05-19T03:43:38.536641235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:43:38.539160117Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:43:38.635458036Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:43:38.635469052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:43:38.635479824Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:43:38.635489854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:43:39.361259491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:43:40.882327594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:43:40.888842752Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:43:41.593994611Z caller=checkpoint.go:372 msg="writing series" size="428 kB"
level=info ts=2026-05-19T03:43:42.975082804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:43:42.977400145Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:43:43.395509526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:43:43.635591192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:43:45.396886135Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-19T03:43:48.635814311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:43:49.200003307Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:43:49.20002618Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-19T03:43:50.107447191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:43:50.110202377Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:43:50.725964791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:43:51.277286409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:43:51.279956559Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T03:43:53.00265988Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-19T03:43:53.63540816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:43:54.963581414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:43:54.968702049Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:43:55.530259101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:43:55.533727999Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:43:56.357968411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:43:56.36063633Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T03:43:56.811320919Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-19T03:43:56.811349009Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=2
level=warn ts=2026-05-19T03:43:58.635689998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:44:00.607616068Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=warn ts=2026-05-19T03:44:03.635310585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:44:03.669445184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="51 kB" avg_uncomp="51 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:44:03.673561479Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-19T03:44:04.410341792Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-19T03:44:07.924487291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:44:07.930599129Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:44:08.213719003Z caller=checkpoint.go:372 msg="writing series" size="576 kB"
level=info ts=2026-05-19T03:44:08.617427131Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:44:08.617974159Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:44:08.635133968Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:44:08.635151833Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:44:08.635160165Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:44:08.635205656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:44:08.699668919Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T03:44:09.363089637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:44:09.549156793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:44:09.55206169Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T03:44:12.016263876Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-19T03:44:13.635909833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:44:14.452085841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:44:14.454955137Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T03:44:15.819078902Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=debug ts=2026-05-19T03:44:15.819103333Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-19T03:44:15.96423592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:44:15.966964667Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:44:17.104054331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:44:17.10678948Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:44:17.560872969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:44:17.56320643Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:44:18.399582043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:44:18.405664813Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:44:18.635487521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:44:18.837973566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49332"
level=debug ts=2026-05-19T03:44:19.622294782Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-19T03:44:20.791570112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:44:20.79422636Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T03:44:23.425198782Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-19T03:44:23.63555533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:44:24.779717692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:44:24.782431409Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:44:27.227146989Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-19T03:44:27.679415907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:44:27.683122188Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T03:44:28.635696179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:44:31.030317496Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-19T03:44:33.635768265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:44:34.832727057Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-19T03:44:36.871144944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:44:36.879158062Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-19T03:44:37.188396399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42822"
level=info ts=2026-05-19T03:44:38.056369247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:44:38.06213373Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:44:38.634831136Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:44:38.634854528Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:44:38.634862062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:44:38.634886146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:44:38.635879084Z caller=checkpoint.go:384 msg="flushing series" totalSize="654 kB" series=5
level=info ts=2026-05-19T03:44:38.642663161Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-19T03:44:38.645399258Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010041991s
level=info ts=2026-05-19T03:44:38.800365154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:44:38.802993761Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
ts=2026-05-19T03:44:39.366139868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:44:39.975231941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:44:39.977790623Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T03:44:40.659670541Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:44:40.992487541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:44:40.995180655Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-19T03:44:43.635567878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:44:47.003097018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:44:47.006138788Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:44:48.433256662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:44:48.435765969Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T03:44:48.63494618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:44:48.841512887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34286"
level=debug ts=2026-05-19T03:44:50.562724177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:44:50.564378604Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T03:44:53.63510012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:44:54.536961665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:44:54.539675532Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T03:44:58.635187279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:45:00.867368366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:45:00.870135626Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:45:01.211112122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="51 kB" avg_uncomp="51 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:45:01.213827033Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-19T03:45:03.635586984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:45:05.909094583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:45:05.912265174Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:45:07.889578623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:45:07.89438197Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T03:45:08.012859297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:45:08.01898559Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:45:08.617423514Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:45:08.61742621Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-19T03:45:08.618770089Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-19T03:45:08.618810893Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976846 n=1
level=debug ts=2026-05-19T03:45:08.618849685Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976846 n=1
level=info ts=2026-05-19T03:45:08.634990456Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:45:08.635006291Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:45:08.635014736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T03:45:08.635014089Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-19T03:45:08.635040367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:45:08.635127414Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-19T03:45:08.635135865Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:45:08.635152581Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.843µs success=true
level=debug ts=2026-05-19T03:45:08.635726357Z caller=checkpoint.go:372 msg="writing series" size="882 kB"
level=debug ts=2026-05-19T03:45:08.902513871Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T03:45:09.368745111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:45:09.997962415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:45:10.002345006Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:45:10.059756796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:45:10.063037745Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:45:12.438585325Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-19T03:45:13.635503029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:45:14.155520647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:45:14.157949671Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T03:45:16.242022316Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-19T03:45:16.242043085Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=3
level=info ts=2026-05-19T03:45:16.477889144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:45:16.480562605Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T03:45:18.635351291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:45:20.044085898Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T03:45:23.170578996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:45:23.173077071Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T03:45:23.635491933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:45:23.847407954Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T03:45:27.283893069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:45:27.290508637Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:45:27.649778693Z caller=checkpoint.go:372 msg="writing series" size="412 kB"
level=warn ts=2026-05-19T03:45:28.635223757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:45:31.45290232Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=debug ts=2026-05-19T03:45:31.452924187Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-19T03:45:33.635092274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:45:34.660725382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:45:34.66327314Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:45:35.255960073Z caller=checkpoint.go:372 msg="writing series" size="646 kB"
level=info ts=2026-05-19T03:45:35.297857277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:45:35.30293141Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:45:35.665396743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:45:37.183561132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T03:45:37.190791522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45962"
level=info ts=2026-05-19T03:45:38.63546121Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:45:38.635484671Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:45:38.635492653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:45:38.635496614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:45:39.058237826Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
ts=2026-05-19T03:45:39.370101742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:45:39.716896293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:45:39.72235961Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:45:39.760409285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:45:39.762862105Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:45:40.825943163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:45:40.829493037Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:45:42.861516954Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-19T03:45:43.20234341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:45:43.204634016Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:45:43.574585648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:45:43.577455621Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:45:43.635433626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:45:46.256561193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="51 kB" avg_uncomp="51 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:45:46.25897661Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-19T03:45:46.664241207Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-19T03:45:46.73552241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:45:46.738422727Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T03:45:48.635425227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:45:50.467121694Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-19T03:45:50.767069029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:45:50.769674329Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:45:52.668775635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:45:52.671239993Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-19T03:45:53.635166653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:45:54.270315131Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=debug ts=2026-05-19T03:45:54.270335965Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-19T03:45:54.35033865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:45:54.354366369Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:45:58.072176998Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-19T03:45:58.635672634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:45:59.062990864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:45:59.065837264Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T03:46:01.883957701Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:46:01.883979869Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-05-19T03:46:02.780902846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:46:02.7834578Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T03:46:03.635576526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:46:05.677751736Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:46:06.346712928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:46:06.353694813Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:46:08.617560143Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:46:08.61797807Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:46:08.635701768Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:46:08.635718395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:46:08.635726239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:46:08.635743503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:46:09.370904634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:46:09.480904752Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-19T03:46:09.93620404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:46:09.941796929Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:46:10.273808393Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:46:13.282166529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="629 B" avg_comp="629 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:46:13.283437254Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=error ts=2026-05-19T03:46:13.287154664Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T03:46:13.635701912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:46:17.086720499Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-19T03:46:18.635789543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:46:18.916218057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:46:18.922581427Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:46:18.922602027Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:46:18.922632259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:46:18.928438039Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:46:20.889812859Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-19T03:46:23.635586264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:46:24.69270057Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T03:46:26.485940823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="51 kB" avg_uncomp="51 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:46:26.48836253Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-19T03:46:28.494791858Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-19T03:46:28.575157669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:46:28.577474335Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T03:46:28.635551877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:46:28.665719345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:46:28.668012234Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:46:28.839552806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:46:29.402850847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:46:29.405063901Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:46:29.730201985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:46:29.733101174Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:46:31.23651732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:46:31.238824001Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:46:31.710430945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:46:31.715419165Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:46:32.297938632Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-19T03:46:33.635623461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:46:34.136975301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:46:34.139386679Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:46:34.970342652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:46:34.976751365Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:46:36.100482824Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-19T03:46:36.786450237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:46:36.788989274Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:46:38.091583532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:46:38.094034841Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T03:46:38.63489964Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:46:38.634897787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:46:38.634922136Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:46:38.634930269Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:46:39.373107085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:46:39.903144395Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T03:46:41.713576453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:46:41.717864218Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:46:43.241140617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:46:43.245192219Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-19T03:46:43.635201674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:46:43.706409838Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-19T03:46:46.082292527Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:46:47.509656788Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-19T03:46:48.635174571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:46:48.85625923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34696"
level=info ts=2026-05-19T03:46:49.875192622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:46:49.878989033Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T03:46:51.312051885Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-19T03:46:51.312072669Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=warn ts=2026-05-19T03:46:53.635156256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:46:53.644269054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:46:53.651328927Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:46:53.651347156Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:46:53.65137238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:46:53.657059035Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:46:54.134106703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:46:54.136677067Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:46:54.501092483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:46:54.504519602Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:46:55.114261171Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-19T03:46:58.63495381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:46:58.917551409Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-19T03:47:02.720883958Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-19T03:47:03.635416059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:47:06.523218081Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-19T03:47:06.974555321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:47:06.979308568Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:47:08.617326234Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:47:08.617824955Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:47:08.635042746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:47:08.635059865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:47:08.635069162Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:47:08.635093714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:47:09.375685599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:47:09.470583299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:47:09.477607747Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:47:10.325681991Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T03:47:11.07778825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="51 kB" avg_uncomp="51 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:47:11.08074836Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e3e1ae5cf%3A19e3e1b7a52%3Af2351dca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-19T03:47:11.080767459Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:47:11.080788199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="121 B" avg_comp="121 B" total_uncomp="184 kB" avg_uncomp="184 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:47:11.083247563Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T03:47:12.859138916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:47:12.867187258Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:47:13.206120671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:47:13.2092297Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:47:13.209248997Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:47:13.20926912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:47:13.211781922Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:47:13.250842009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:47:13.252967542Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T03:47:13.635874584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:47:13.670026176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:47:13.67243026Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T03:47:14.129209124Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-19T03:47:14.649937989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:47:16.717572636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:47:17.93217565Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=debug ts=2026-05-19T03:47:17.932199028Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-19T03:47:18.635304061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:47:19.67803057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:47:19.680592392Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:47:21.734382498Z caller=checkpoint.go:372 msg="writing series" size="483 kB"
level=info ts=2026-05-19T03:47:23.627211278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:47:23.630173423Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:47:23.630191319Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:47:23.630210719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:47:23.632064091Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T03:47:23.635144937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:47:24.471330733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:47:24.473799442Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T03:47:25.537622973Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-05-19T03:47:26.165414734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:47:26.168053057Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T03:47:27.523933436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:47:27.526594203Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-19T03:47:28.635225825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:47:29.027204154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:47:29.029504508Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:47:29.029521367Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:47:29.029560392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:47:29.03141641Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:47:29.144669303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:47:29.14897934Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:47:29.148993607Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:47:29.330575311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:47:29.335632811Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:47:29.339728901Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-19T03:47:31.700230086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:47:31.703365124Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:47:33.143256737Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=warn ts=2026-05-19T03:47:33.635372319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:47:33.991223743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:47:33.993654239Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T03:47:34.3154108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:47:34.317793401Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:47:35.0775736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:47:35.079950505Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:47:36.890079296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:47:36.896838441Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:47:36.946353585Z caller=checkpoint.go:372 msg="writing series" size="567 kB"
level=debug ts=2026-05-19T03:47:36.946372674Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-19T03:47:38.635851091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:47:38.635875002Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:47:38.635882769Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:47:38.635871932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:47:39.376765435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:47:40.357931951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:47:40.361925578Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:47:40.748120845Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-19T03:47:41.223032882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:47:41.228360842Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:47:42.538924762Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:47:43.635784201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:47:44.551753826Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-19T03:47:45.829934116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:47:45.834215033Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:47:47.219580846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:47:47.222332027Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T03:47:48.354338827Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=warn ts=2026-05-19T03:47:48.635105278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:47:50.69151983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:47:50.698335502Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:47:51.870801627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:47:51.873172029Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-19T03:47:51.873191818Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:47:51.873211573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:47:51.875265283Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:47:51.8973316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:47:51.900254753Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:47:51.900267746Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:47:52.048899631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:47:52.050883412Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T03:47:52.157115446Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-19T03:47:52.17425564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:47:52.177288475Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T03:47:53.635179516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:47:54.038695766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:47:55.73192429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:47:55.734227947Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:47:55.734242921Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:47:55.734260103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:47:55.736100815Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T03:47:55.959913465Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=info ts=2026-05-19T03:47:56.95652707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:47:56.961128111Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:47:58.635628156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:47:59.762793711Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T03:48:02.330724396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:48:02.333188415Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:48:03.565479176Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-19T03:48:03.635548398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:48:04.963096035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:48:04.968337984Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:48:05.151904149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:48:05.154403429Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-19T03:48:05.154421033Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:48:05.154439128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:48:05.156447302Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-19T03:48:07.201727669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39780"
level=debug ts=2026-05-19T03:48:07.36877833Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-19T03:48:08.617429768Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:48:08.617857581Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:48:08.635571634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:48:08.635586813Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:48:08.635594005Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:48:08.635600459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:48:09.378420983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:48:10.271977804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:48:10.275618865Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:48:11.171024072Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-19T03:48:11.50490437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:48:11.507560993Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:48:11.507585046Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:48:11.507610446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:48:11.509789667Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:48:11.973045444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:48:11.975615378Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T03:48:13.635703011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:48:13.739008917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:48:13.74142829Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-19T03:48:13.97177933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38378"
level=debug ts=2026-05-19T03:48:14.974179556Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-19T03:48:16.83972322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:48:16.844321639Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:48:18.041563392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:48:18.044167637Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:48:18.044184259Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:48:18.044204833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:48:18.046193806Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:48:18.289895414Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:48:18.635624095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:48:18.776985383Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-19T03:48:19.081189982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:48:19.083884936Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:48:19.083902415Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:48:19.083920319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:48:19.085964686Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:48:21.870941669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:48:21.873775047Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:48:22.436345678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:48:22.440286099Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:48:22.440304834Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:48:22.440337403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:48:22.443759714Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:48:22.579042376Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-19T03:48:23.436575646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:48:23.439494053Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:48:23.439513395Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:48:23.439548023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:48:23.441663221Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T03:48:23.63582084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:48:25.56858846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:48:25.570965291Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:48:26.382467327Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-19T03:48:26.647354559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:48:26.653333266Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:48:28.635706397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:48:30.018408895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:48:30.021171508Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:48:30.186517976Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-19T03:48:30.186554469Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=14
level=info ts=2026-05-19T03:48:30.612995494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:48:30.615820871Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:48:30.615837291Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:48:30.615855764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:48:30.618215954Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T03:48:33.635379404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:48:33.855634412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:48:33.858303436Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T03:48:33.987516252Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T03:48:34.997912005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:48:35.000599513Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:48:35.301444152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:48:35.305102114Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:48:35.604881571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:48:35.608933358Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:48:37.791084791Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-19T03:48:38.63541075Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:48:38.635433638Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:48:38.635424467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:48:38.635441152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:48:39.379948957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:48:39.908023626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:48:39.910688984Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:48:41.399352613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:48:41.40215795Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T03:48:41.593923735Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=warn ts=2026-05-19T03:48:43.635556793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:48:43.972987126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36242"
level=info ts=2026-05-19T03:48:43.999368502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:48:44.002009703Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T03:48:45.396487781Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-19T03:48:45.800661731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:48:45.803445491Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:48:45.803463377Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:48:45.803482918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:48:45.805493748Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:48:47.625711223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:48:47.63012754Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T03:48:48.635206214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:48:48.660400636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:48:48.664875039Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:48:48.75117772Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:48:49.199236202Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:48:49.19925532Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-19T03:48:49.649221933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:48:49.651247035Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T03:48:49.990472053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:48:49.991921519Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T03:48:50.188523309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:48:50.19457962Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:48:53.002329397Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-19T03:48:53.635229223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:48:53.646304695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:48:53.649595549Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:48:55.172062522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:48:55.176209487Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T03:48:56.147795452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:48:56.150437669Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:48:56.259783719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:48:56.262197973Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T03:48:56.811340383Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-19T03:48:56.811361613Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=2
level=warn ts=2026-05-19T03:48:58.635805362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:48:59.99493472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:48:59.99736778Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:49:00.373924172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:49:00.376372358Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T03:49:00.608227061Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-19T03:49:00.673554114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:49:00.675811089Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-19T03:49:03.635486509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:49:04.307670963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:49:04.311442198Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:49:04.410844265Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=debug ts=2026-05-19T03:49:08.212999049Z caller=checkpoint.go:372 msg="writing series" size="524 kB"
level=info ts=2026-05-19T03:49:08.617301701Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:49:08.617463614Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:49:08.635106877Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:49:08.635126799Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:49:08.635136694Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:49:08.635147969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:49:09.320784709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:49:09.327012398Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-19T03:49:09.381262929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:49:11.552566613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:49:11.555338939Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T03:49:12.015959993Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-19T03:49:13.634869864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:49:14.122194148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:49:14.125146753Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T03:49:15.8193351Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=debug ts=2026-05-19T03:49:15.819358477Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-19T03:49:18.63565354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:49:18.873692999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48822"
level=debug ts=2026-05-19T03:49:19.621607196Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=debug ts=2026-05-19T03:49:21.587062385Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:49:21.787701402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:49:21.790777774Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:49:22.930904905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:49:22.933787518Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:49:23.424598911Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-19T03:49:23.63504688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:49:25.761750744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:49:27.227079009Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T03:49:28.558843091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:49:28.561340861Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T03:49:28.635579342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:49:31.030422414Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-19T03:49:32.198964944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:49:32.202294373Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T03:49:33.634931127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:49:34.833022027Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-19T03:49:36.737191897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:49:36.740550989Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:49:38.634984066Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:49:38.634989024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:49:38.635013476Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:49:38.635028247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:49:38.636073485Z caller=checkpoint.go:384 msg="flushing series" totalSize="518 kB" series=5
level=info ts=2026-05-19T03:49:38.642370907Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-19T03:49:38.645103297Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009839632s
ts=2026-05-19T03:49:39.382751618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:49:40.396826349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:49:40.399249886Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:49:41.804261245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:49:41.80734226Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:49:42.019998253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:49:42.025092188Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:49:43.239325668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:49:43.241777347Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:49:43.472361569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:49:43.474776165Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-19T03:49:43.635156505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:49:45.759545305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:49:45.766165828Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:49:48.635419179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:49:49.371393313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:49:49.375472589Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T03:49:49.488838892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:49:49.491196339Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T03:49:50.505304325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:49:50.508288698Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:49:52.393242319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:49:52.395861422Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T03:49:53.635140925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:49:57.015963163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:49:58.635158933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:50:00.513924937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:50:00.516435475Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T03:50:00.958729064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:50:00.965929176Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:50:03.635765177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:50:06.108984934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:50:06.112174523Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-19T03:50:07.207245337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:43850"
level=info ts=2026-05-19T03:50:08.617469355Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:50:08.61792023Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:50:08.635398581Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-19T03:50:08.635400096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:50:08.635414503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:50:08.635432558Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:50:08.635441989Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T03:50:08.635513374Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-19T03:50:08.63552091Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:50:08.635552988Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=27.244µs success=true
level=debug ts=2026-05-19T03:50:08.636128135Z caller=checkpoint.go:372 msg="writing series" size="958 kB"
level=info ts=2026-05-19T03:50:08.985269522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:50:08.987674557Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-19T03:50:09.383743112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:50:10.609764854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:50:10.61258969Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T03:50:12.439355191Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-19T03:50:13.635710197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:50:16.242340289Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-19T03:50:16.242361699Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=warn ts=2026-05-19T03:50:18.634901833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:50:19.362562821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:50:20.044805437Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T03:50:20.564054722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:50:20.567493853Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T03:50:23.635551234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:50:23.847826894Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T03:50:24.169062729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:50:24.171721427Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T03:50:25.540303052Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:50:25.613582916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:50:25.615755418Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:50:27.650995464Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=info ts=2026-05-19T03:50:27.960584743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:50:27.962906022Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T03:50:28.635641541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:50:30.174735957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:50:30.180072843Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T03:50:31.453300441Z caller=checkpoint.go:372 msg="writing series" size="909 kB"
level=debug ts=2026-05-19T03:50:31.453323104Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-19T03:50:31.967201655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:50:31.969835653Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T03:50:33.635692793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:50:34.402765875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:50:34.405261053Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T03:50:35.2562026Z caller=checkpoint.go:372 msg="writing series" size="646 kB"
level=info ts=2026-05-19T03:50:36.740084424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:50:36.744974572Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:50:37.105829821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:50:37.10841518Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T03:50:37.946735804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:50:37.949186482Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:50:38.634876954Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:50:38.634878254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:50:38.63489985Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:50:38.634907785Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:50:39.05930877Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
ts=2026-05-19T03:50:39.386458525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:50:41.44317021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:50:41.447673065Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T03:50:42.568587705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:50:42.574837133Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:50:42.861885763Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-19T03:50:43.635772702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:50:45.368479196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:50:45.371706128Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:50:45.383739974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:50:45.389568162Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:50:46.664524766Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-19T03:50:48.635618267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:50:50.467381614Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-19T03:50:52.967194324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:50:52.971359819Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T03:50:53.635577378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:50:54.27074407Z caller=checkpoint.go:372 msg="writing series" size="395 kB"
level=debug ts=2026-05-19T03:50:54.270765961Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-19T03:50:56.913727841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:50:56.91664605Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T03:50:57.391839885Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:50:58.07291369Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-19T03:50:58.635848464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:51:00.361093825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="629 B" avg_comp="629 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:51:00.365559708Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:51:01.881268533Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:51:01.881291619Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=warn ts=2026-05-19T03:51:03.63592279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:51:05.678559354Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-19T03:51:07.134643109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:51:07.137692493Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:51:07.349183985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:51:07.353171356Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:51:08.617445859Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:51:08.617838331Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:51:08.635064788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:51:08.635081944Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:51:08.635089605Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:51:08.635106532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:51:09.388567174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:51:09.481658767Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-19T03:51:09.902912267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:51:09.905249931Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T03:51:10.637267777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:51:12.836191856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:51:12.838624795Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:51:13.283784567Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:51:13.634909868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:51:15.826582934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:51:15.82927861Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T03:51:17.086897748Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-19T03:51:18.635121669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:51:18.887551783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37402"
level=debug ts=2026-05-19T03:51:20.889948273Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-19T03:51:21.65272402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:51:21.655800075Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:51:21.697863211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:51:21.700400606Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:51:22.812713293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:51:22.816722643Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-19T03:51:23.635814181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:51:24.692167694Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T03:51:28.398552883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:51:28.401432189Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T03:51:28.495822249Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-19T03:51:28.610448301Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:51:28.635465013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:51:31.753473665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:51:31.763453532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:51:31.855817452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:51:31.86062109Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:51:32.297761905Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-19T03:51:33.635171291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:51:35.930491491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:51:35.933244875Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:51:36.004325258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:51:36.006768615Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T03:51:36.101119811Z caller=checkpoint.go:372 msg="writing series" size="657 B"
ts=2026-05-19T03:51:37.214024069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:57384"
level=info ts=2026-05-19T03:51:37.978660512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:51:37.981371536Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T03:51:38.554094674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:51:38.560424274Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:51:38.635680973Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:51:38.635701988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:51:38.635710104Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:51:38.635700833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:51:39.065112898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:51:39.068523374Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-19T03:51:39.389517711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:51:39.90370199Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-19T03:51:43.635484125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:51:43.706679157Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-19T03:51:43.98871041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52542"
level=info ts=2026-05-19T03:51:46.384762118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:51:46.387840965Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:51:47.509161463Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-19T03:51:47.960428109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:51:47.963460069Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T03:51:48.635215998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:51:48.891058434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60254"
level=debug ts=2026-05-19T03:51:51.312033916Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-19T03:51:51.312057266Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=13
level=info ts=2026-05-19T03:51:52.248579178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:51:52.251116769Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-19T03:51:53.635601301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:51:55.114959946Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-19T03:51:58.332750423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:51:58.339495753Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T03:51:58.635901275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:51:58.918178202Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-19T03:52:01.432870175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:52:01.435581543Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:52:01.613256508Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:52:02.720663905Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T03:52:03.228968299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:52:03.233992803Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-19T03:52:03.635729952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:52:04.827070171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:52:04.829980646Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T03:52:06.523253651Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-19T03:52:08.558836822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:52:08.562659653Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T03:52:08.589714397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:52:08.591747464Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T03:52:08.61719254Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:52:08.618251329Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T03:52:08.635209328Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:52:08.635226651Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:52:08.635235717Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:52:08.635246737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:52:09.008968628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:52:09.391945226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:52:10.326459376Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-19T03:52:13.635434463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:52:13.990311714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44322"
level=info ts=2026-05-19T03:52:14.014135986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:52:14.016719552Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T03:52:14.128918537Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-19T03:52:17.931936677Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=warn ts=2026-05-19T03:52:18.635456659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:52:21.735034353Z caller=checkpoint.go:372 msg="writing series" size="612 kB"
level=debug ts=2026-05-19T03:52:21.735057348Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-05-19T03:52:22.703513758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:52:22.709037707Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:52:23.635120011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:52:24.398624357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:52:24.405107796Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:52:25.538132543Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-05-19T03:52:26.682417988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:52:26.685628286Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T03:52:28.209608692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:52:28.212294834Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T03:52:28.635405163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:52:29.090935084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:52:29.094564919Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:52:29.340762169Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-19T03:52:30.294975427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:52:30.301734957Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:52:33.143244936Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=info ts=2026-05-19T03:52:33.602299868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:52:33.607703144Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:52:33.635594533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:52:33.704726353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:52:33.70737734Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T03:52:34.661560499Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:52:34.661609026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="320 B" avg_comp="320 B" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" idle=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:52:34.66405672Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T03:52:36.946460018Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=debug ts=2026-05-19T03:52:36.946480958Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-19T03:52:37.217066175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:41068"
level=info ts=2026-05-19T03:52:38.634968429Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:52:38.634992558Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:52:38.635000382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:52:38.635002344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:52:39.395052098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:52:40.748714287Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=warn ts=2026-05-19T03:52:43.635644404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:52:43.756982456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:52:43.759800785Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:52:44.231353787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:52:44.234074019Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T03:52:44.551822551Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-19T03:52:48.355079718Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-19T03:52:48.368099402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:52:48.37123694Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-19T03:52:48.635677309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:52:48.970415903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:52:48.973328608Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:52:50.54246131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:52:50.544963277Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T03:52:51.460726221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:52:51.463078675Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:52:51.535214534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:52:51.5383979Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:52:52.157747382Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-19T03:52:53.635918542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:52:55.960641362Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=info ts=2026-05-19T03:52:57.922147227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:52:57.925445171Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:52:58.107854159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:52:58.110268941Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T03:52:58.635637015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:52:59.76333544Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T03:52:59.849703521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:52:59.852128735Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:53:00.787305685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:53:00.788694896Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:53:02.747325871Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:53:02.747383256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:53:02.750206897Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-19T03:53:03.565721655Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-19T03:53:03.635355071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:53:05.756741762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:53:05.762591596Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-19T03:53:07.217996245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34784"
level=debug ts=2026-05-19T03:53:07.3687998Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-19T03:53:08.617104138Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:53:08.61815903Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T03:53:08.634945077Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:53:08.634964629Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:53:08.634972878Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:53:08.635030684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:53:09.398825223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:53:11.171592105Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-19T03:53:11.310182336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:53:11.31694353Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:53:11.316962663Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:53:11.316985205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:53:11.322666746Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:53:13.635440323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:53:14.437273292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:53:14.439925152Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-19T03:53:14.974434147Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-05-19T03:53:15.237291752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:53:15.239599068Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:53:16.925456221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:53:16.930890572Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:53:18.497583102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:53:18.499948287Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T03:53:18.635586508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:53:18.777065364Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
ts=2026-05-19T03:53:18.900846426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57776"
level=info ts=2026-05-19T03:53:19.899375163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:53:19.901838453Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T03:53:21.347251612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:53:21.353419347Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:53:21.35343866Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:53:21.353463619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:53:21.359624416Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T03:53:22.064737249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:53:22.071639789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:53:22.579968827Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-19T03:53:23.190669844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:53:23.193390433Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-19T03:53:23.635043428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:53:24.624005193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:53:24.626606398Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T03:53:25.776421205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:53:25.778931398Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T03:53:26.382778199Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-19T03:53:28.635570041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:53:30.186916681Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-19T03:53:30.186939245Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=14
level=info ts=2026-05-19T03:53:32.160530072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:53:32.163559063Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T03:53:32.778343349Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:53:33.375935751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:53:33.37924618Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:53:33.635818732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:53:33.987874116Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T03:53:34.270489504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:53:34.273027059Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:53:34.614037866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:53:34.616476825Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:53:35.970485513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:53:35.975260314Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T03:53:37.399347504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:53:37.406697347Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:53:37.790831453Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-19T03:53:38.635418901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:53:38.635428208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:53:38.635441216Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:53:38.635449219Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T03:53:39.003836303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:53:39.006633928Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:53:39.006656044Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:53:39.006678022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:53:39.009023034Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-19T03:53:39.401146148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:53:41.078977744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:53:41.081462289Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:53:41.593852151Z caller=checkpoint.go:372 msg="writing series" size="437 kB"
level=warn ts=2026-05-19T03:53:43.635719919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:53:44.569550363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:53:44.572063348Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T03:53:45.396963632Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-19T03:53:46.927371274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:53:46.934403385Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:53:48.113735795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:53:48.116262217Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:53:48.131235773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T03:53:48.635662884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:53:49.199757435Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:53:49.19978223Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-19T03:53:51.289120253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:53:51.292903265Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:53:53.002212742Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-19T03:53:53.635433735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:53:53.942582966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:53:53.945322411Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:53:56.651555126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:53:56.655939872Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:53:56.813406442Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-19T03:53:56.813427948Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=2
level=warn ts=2026-05-19T03:53:58.635524977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:54:00.60849438Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=info ts=2026-05-19T03:54:02.343317275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:54:02.351278448Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:54:03.635520005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:54:03.714171645Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:54:03.714231165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="368 B" avg_comp="368 B" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" idle=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T03:54:03.716774444Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=debug ts=2026-05-19T03:54:04.411338771Z caller=checkpoint.go:372 msg="writing series" size="342 kB"
level=info ts=2026-05-19T03:54:06.327918359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:54:06.332574919Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:54:06.450036675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:54:06.452888856Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T03:54:07.073587987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:54:07.078340565Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:54:07.078358183Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:07.078379516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:54:07.08261636Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:54:07.484655298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:54:07.488005034Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T03:54:08.213432423Z caller=checkpoint.go:372 msg="writing series" size="405 kB"
level=info ts=2026-05-19T03:54:08.617632296Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:54:08.618006358Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:54:08.63575821Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:54:08.635783741Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:54:08.635787357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:54:08.635800082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T03:54:09.402578096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T03:54:12.016732641Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-19T03:54:12.68445834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:54:12.687367196Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-19T03:54:12.68738837Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:12.687409146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:54:12.689500187Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T03:54:13.635575847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:54:13.996669516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58976"
level=info ts=2026-05-19T03:54:14.51339286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:54:14.516183686Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T03:54:15.819500871Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-19T03:54:15.819523482Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-19T03:54:16.589671512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:54:16.592617381Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:54:16.592636643Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:16.59265654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:54:16.59471166Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T03:54:17.735452309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:54:17.73850175Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-19T03:54:18.635873443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:54:19.622237009Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=info ts=2026-05-19T03:54:21.813857368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:54:21.820145112Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:54:23.289977217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T03:54:23.292828026Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=debug ts=2026-05-19T03:54:23.425375591Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-19T03:54:23.53367301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:54:23.536125091Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:54:23.536143685Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:23.536165791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:54:23.53801554Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T03:54:23.635310843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:54:24.371565805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:54:24.374004782Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T03:54:27.227855327Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-19T03:54:28.635390783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:54:29.067339105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:54:29.070277234Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:54:29.070299591Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:29.070322573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:54:29.072502089Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T03:54:29.176826937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:54:29.17946799Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:54:29.17948403Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:29.37095053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:54:29.373737798Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:54:29.692738446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:54:29.695426975Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:54:29.758657866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:54:29.761231123Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-19T03:54:29.761249838Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:29.761269593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:54:29.763380775Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:54:29.95393759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:54:29.957169589Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:54:29.957187886Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:30.061758399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:54:30.065012437Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:54:31.031050207Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-19T03:54:31.296662619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:54:31.300485383Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:54:31.300506137Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:31.300526327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:54:31.302628722Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:54:31.552288634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:54:31.554889916Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T03:54:32.27200164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:54:32.274719262Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-19T03:54:33.635013142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:54:34.833608758Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-19T03:54:37.0999793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:54:37.102529791Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-19T03:54:37.221814022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45944"
level=info ts=2026-05-19T03:54:37.818337638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:54:37.82464788Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:54:37.845775179Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:54:38.634847017Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:54:38.634870297Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:54:38.634878356Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:54:38.634873352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:54:38.635933055Z caller=checkpoint.go:384 msg="flushing series" totalSize="551 kB" series=5
level=info ts=2026-05-19T03:54:38.642224853Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-19T03:54:38.644809454Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009135953s
level=debug ts=2026-05-19T03:54:39.215831515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:54:39.404351913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:54:41.905196523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:54:41.907718699Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T03:54:42.862429441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:54:42.868492122Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:54:43.635900641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:54:43.998446187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54230"
level=info ts=2026-05-19T03:54:46.256447039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:54:46.258957469Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:54:46.293035015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:54:46.295145343Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:54:46.795526109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:54:46.798490664Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:54:48.635671266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:54:49.40197634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:54:49.404359342Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:54:50.621763959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:54:50.625083983Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:54:50.625102254Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:50.625121716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:54:50.627942059Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:54:50.642082389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:54:50.643869878Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:54:50.643881667Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:50.925586697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:54:50.92793052Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:54:52.065078626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:54:52.068927724Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:54:52.944450514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:54:52.949088307Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:54:53.635110504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:54:54.868190744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T03:54:54.870549897Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T03:54:54.890578552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:54:54.89271568Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:54:54.89273221Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:54.892752138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:54:54.894701571Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:54:55.802259082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:54:55.806382245Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:54:55.806402508Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:55.806425899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:54:55.810443767Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T03:54:57.804980659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:54:57.807527549Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T03:54:57.984958323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:54:57.98736792Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:54:57.987386641Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:54:57.987405562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:54:57.989202328Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T03:54:58.63525028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:55:02.871433453Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T03:55:03.05694255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:55:03.05958947Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T03:55:03.393319375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:55:03.395865651Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T03:55:03.635448627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:55:06.112512176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:55:06.116849967Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:55:06.116868884Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:55:06.11688959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:55:06.118717161Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:55:06.684515216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:55:06.68686818Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T03:55:07.192527764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:55:07.195934918Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:55:07.615269282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:55:07.617676264Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:55:08.617340805Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:55:08.617801285Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:55:08.634685582Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T03:55:08.634822595Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-19T03:55:08.634831545Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T03:55:08.634831619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:55:08.634847475Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:55:08.634850608Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.862µs success=true
level=info ts=2026-05-19T03:55:08.634855342Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:55:08.634874162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:55:08.635357027Z caller=checkpoint.go:372 msg="writing series" size="812 kB"
level=info ts=2026-05-19T03:55:08.977984425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:55:08.980602875Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:55:09.032101423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:55:09.036026511Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-19T03:55:09.405218042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:55:09.762074311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:55:09.764495928Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:55:10.308996525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:55:10.312377498Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T03:55:12.4385603Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-19T03:55:13.153526964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:55:13.155967955Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T03:55:13.635048524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:55:15.532128332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:55:15.534958448Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T03:55:16.24176204Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-19T03:55:16.241786641Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=warn ts=2026-05-19T03:55:18.63540336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:55:20.04415982Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T03:55:20.61588612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:55:20.619844805Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:55:22.063790565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:55:22.066365057Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:55:23.484186934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:55:23.491905306Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:55:23.635077895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:55:23.847022338Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-19T03:55:27.650043072Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=info ts=2026-05-19T03:55:27.657991844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:55:27.661326836Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T03:55:28.361064691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:55:28.364042565Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T03:55:28.635500042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:55:31.45310939Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-19T03:55:31.453130411Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-19T03:55:33.635181776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:55:34.313020096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:55:35.061463556Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:55:35.255346709Z caller=checkpoint.go:372 msg="writing series" size="646 kB"
level=warn ts=2026-05-19T03:55:38.635132702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:55:38.635132699Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:55:38.6351584Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:55:38.635168403Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:55:39.058109266Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
ts=2026-05-19T03:55:39.407973213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:55:39.885968358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:55:39.889337613Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T03:55:41.003266981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:55:41.005809516Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T03:55:41.44909005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:55:41.451574641Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:55:41.731401226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T03:55:41.733870098Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T03:55:42.714679021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:55:42.717122202Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:55:42.731171643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:55:42.733318961Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:55:42.839718739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:55:42.845758049Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:55:42.860824053Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-19T03:55:43.63550893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:55:44.570859382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:55:44.574938853Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T03:55:45.08132464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:55:45.0846826Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:55:46.608185099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:55:46.610702461Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:55:46.663421622Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-19T03:55:47.766554874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:55:47.76924765Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-19T03:55:48.635702749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:55:50.466425332Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-19T03:55:52.255597738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:55:52.260848798Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T03:55:53.63588334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:55:54.269714541Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=debug ts=2026-05-19T03:55:54.269745012Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-19T03:55:56.332204977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:55:56.33673863Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T03:55:58.072161026Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-19T03:55:58.635818006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:55:59.634010967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:55:59.637330578Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T03:56:01.883209269Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:56:01.883234636Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-05-19T03:56:02.723043373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:56:02.726057908Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:56:02.896343712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:56:02.89940281Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:56:03.635104513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:56:04.274466975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:56:04.277493805Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T03:56:04.445160955Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:56:05.677579479Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
ts=2026-05-19T03:56:07.23032368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:46096"
level=info ts=2026-05-19T03:56:08.617371932Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:56:08.617392133Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:56:08.635827813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T03:56:08.635831046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:56:08.635846809Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:56:08.635855034Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:56:09.060692781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:56:09.410063666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:56:09.481018578Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-19T03:56:13.283360185Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T03:56:13.635192347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:56:15.169003601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="629 B" avg_comp="629 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:56:15.173447081Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:56:17.086069271Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-19T03:56:18.634988819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:56:18.920709202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37036"
level=info ts=2026-05-19T03:56:19.010505056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:56:19.013339426Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:56:19.039987331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:56:19.046213536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:56:20.889378796Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-19T03:56:23.399247528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T03:56:23.401966484Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=warn ts=2026-05-19T03:56:23.635817764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:56:23.856278434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:56:23.858948046Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:56:24.233259594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:56:24.236422946Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T03:56:24.574187097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:56:24.580376236Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:56:24.691918637Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T03:56:24.763116841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:56:24.765697037Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T03:56:27.870751248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:56:27.87490886Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T03:56:28.03741087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:56:28.04005255Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T03:56:28.495208853Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=warn ts=2026-05-19T03:56:28.634998083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:56:32.297488592Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-19T03:56:33.635962334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:56:34.495295172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:56:34.500396269Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:56:36.100690088Z caller=checkpoint.go:372 msg="writing series" size="657 B"
ts=2026-05-19T03:56:37.231811851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40938"
level=info ts=2026-05-19T03:56:38.635349928Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:56:38.635380223Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:56:38.635389202Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:56:38.63538537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:56:39.146140303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:56:39.149243568Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-19T03:56:39.412986714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:56:39.793493233Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:56:39.902873608Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T03:56:41.236749941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:56:41.241122723Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:56:43.033101713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:56:43.03746843Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T03:56:43.603861045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:56:43.606631241Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-19T03:56:43.635218465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:56:43.706438661Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-19T03:56:44.088662912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:56:44.091668462Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T03:56:44.555329872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:56:44.558873705Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:56:44.850949788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:56:44.853527047Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:56:47.508722828Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=info ts=2026-05-19T03:56:48.490891554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:56:48.494131489Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T03:56:48.63531038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:56:51.312092185Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=debug ts=2026-05-19T03:56:51.312118966Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-05-19T03:56:52.249651904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:56:52.253147494Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T03:56:53.635747339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:56:55.114493477Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-19T03:56:58.634897708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:56:58.917241408Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-19T03:57:00.705566242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:57:00.707143396Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T03:57:02.055789723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:57:02.058370425Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T03:57:02.720377297Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T03:57:02.949749634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:57:02.952478471Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-19T03:57:03.635246391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:57:03.845753876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:57:03.853785371Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T03:57:06.431582575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:57:06.434210393Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T03:57:06.522411188Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-19T03:57:06.742222976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:57:06.751507541Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-19T03:57:07.23349261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34080"
level=info ts=2026-05-19T03:57:08.511020443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:57:08.515229035Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:57:08.617353005Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T03:57:08.617602935Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:57:08.63520262Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:57:08.635221552Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:57:08.635228993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:57:08.635279327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:57:08.893040223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T03:57:08.896061589Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
ts=2026-05-19T03:57:09.415493071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:57:10.325741144Z caller=checkpoint.go:372 msg="writing series" size="275 kB"
level=info ts=2026-05-19T03:57:11.187263478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:57:11.189998731Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T03:57:12.115058952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:57:13.635557129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:57:13.858660466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:57:13.861651582Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-19T03:57:14.128897882Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-19T03:57:15.723519556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:57:15.726582959Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T03:57:17.931114783Z caller=checkpoint.go:372 msg="writing series" size="496 kB"
level=debug ts=2026-05-19T03:57:17.931137465Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-19T03:57:18.635808969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:57:21.734340682Z caller=checkpoint.go:372 msg="writing series" size="502 kB"
level=warn ts=2026-05-19T03:57:23.635288063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:57:24.185002386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:57:24.187952365Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T03:57:24.237056176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:57:24.239785711Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T03:57:25.536999129Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=info ts=2026-05-19T03:57:27.235885167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:57:27.239234938Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:57:27.377501038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:57:27.381751293Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-19T03:57:28.635418635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:57:29.339792053Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=info ts=2026-05-19T03:57:29.352879651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:57:29.357077801Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:57:30.541248206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:57:30.543967715Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:57:31.371278422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:57:31.376309565Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:57:33.142372759Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=warn ts=2026-05-19T03:57:33.635579919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:57:36.946288074Z caller=checkpoint.go:372 msg="writing series" size="748 kB"
level=debug ts=2026-05-19T03:57:36.946313852Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=info ts=2026-05-19T03:57:38.607437997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:57:38.611055652Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:57:38.635824314Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:57:38.635849779Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:57:38.635856964Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:57:38.635866064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:57:39.417405555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:57:40.598059759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:57:40.600859302Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T03:57:40.748563553Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-19T03:57:41.187787412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:57:41.190779009Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T03:57:41.572683314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:57:41.576224794Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:57:42.097061773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:57:43.635303091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:57:44.013715008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37944"
level=debug ts=2026-05-19T03:57:44.550857044Z caller=checkpoint.go:372 msg="writing series" size="8.5 kB"
level=info ts=2026-05-19T03:57:47.603244674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:57:47.60940582Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T03:57:48.354425392Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=warn ts=2026-05-19T03:57:48.635114979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:57:48.930583545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54184"
level=debug ts=2026-05-19T03:57:49.69067427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T03:57:52.156519572Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-19T03:57:52.314110826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:57:52.31657152Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:57:52.639037815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:57:52.645324927Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T03:57:53.635576241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:57:55.92274688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:57:55.925664357Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T03:57:55.959875362Z caller=checkpoint.go:372 msg="writing series" size="413 kB"
level=warn ts=2026-05-19T03:57:58.635688433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:57:59.76201233Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T03:58:00.685753823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:58:00.688327646Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:58:01.474388471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:58:01.477123558Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T03:58:01.888355597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T03:58:01.890798976Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T03:58:02.011226174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:58:02.014004462Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T03:58:03.565205063Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-19T03:58:03.635200968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:58:06.251898837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:58:06.25449837Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:58:07.26265873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:58:07.26503774Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T03:58:07.368317566Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-19T03:58:08.547940579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:58:08.61750756Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:58:08.617518914Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T03:58:08.63557851Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:58:08.63560181Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:58:08.635609463Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:58:08.635618121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T03:58:09.418787826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:58:09.481441142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:58:09.484219995Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-19T03:58:11.171104189Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-19T03:58:13.635106838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:58:14.973219439Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-19T03:58:15.677326229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:58:15.680504818Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:58:15.914242012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:58:15.916847129Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T03:58:18.635451828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:58:18.776774258Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-19T03:58:18.943460844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:58:18.946453824Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T03:58:21.126400698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:58:21.132703361Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T03:58:22.476412686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:58:22.481085801Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:58:22.57938172Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-19T03:58:23.635132144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:58:23.828272128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:58:23.830881092Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T03:58:26.381945581Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=debug ts=2026-05-19T03:58:26.381967155Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-05-19T03:58:26.746911571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:58:26.752019419Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:58:28.132824295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:58:28.138977471Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:58:28.635867704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:58:30.185724264Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-19T03:58:30.185746804Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
level=info ts=2026-05-19T03:58:32.723233745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:58:32.726687369Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-19T03:58:33.635898327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:58:33.987236191Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T03:58:34.909180736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:58:34.916235205Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:58:37.790853193Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=warn ts=2026-05-19T03:58:38.635461371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:58:38.635476812Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:58:38.635506045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:58:38.635516377Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T03:58:39.020938837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:58:39.420658931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T03:58:39.902595107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:58:39.905003267Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T03:58:40.422972419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:58:40.426061814Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T03:58:41.265448239Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T03:58:41.593340012Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=warn ts=2026-05-19T03:58:43.635500504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:58:43.981976821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:58:43.984687639Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-19T03:58:44.01839234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45690"
level=info ts=2026-05-19T03:58:44.970294267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:58:44.973032654Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T03:58:45.396385779Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=warn ts=2026-05-19T03:58:48.635482158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:58:49.199774374Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T03:58:49.199797673Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-19T03:58:51.475118251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:58:51.480846539Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:58:52.202861128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:58:52.205876626Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:58:52.205901526Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:58:52.20592809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:58:52.20816537Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T03:58:53.00146237Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-19T03:58:53.634885901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:58:56.526748468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:58:56.529862488Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T03:58:56.820488193Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-19T03:58:56.820513156Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-05-19T03:58:57.66925178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:58:57.672460566Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T03:58:58.635381309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:58:59.650678168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T03:58:59.653791867Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=debug ts=2026-05-19T03:59:00.607002125Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-19T03:59:01.444511744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:59:01.447938067Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:59:03.075799905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:59:03.078904374Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T03:59:03.635211419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:59:04.205867412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:59:04.208580258Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-19T03:59:04.410663743Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=info ts=2026-05-19T03:59:04.638517545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:59:04.641339953Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T03:59:05.476484919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T03:59:05.479007307Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T03:59:05.937418436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T03:59:05.942213785Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T03:59:08.213152935Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=debug ts=2026-05-19T03:59:08.617062536Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T03:59:08.618130237Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T03:59:08.635242057Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:59:08.6352639Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T03:59:08.635272139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T03:59:08.635309336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:59:09.375249666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T03:59:09.378363407Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-19T03:59:09.422000651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T03:59:10.548814785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:59:10.551648829Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T03:59:12.015972069Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-19T03:59:13.635085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:59:13.86147377Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-19T03:59:14.019802625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34696"
level=debug ts=2026-05-19T03:59:15.818567708Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-19T03:59:15.818595698Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-19T03:59:16.05919123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:59:16.066252172Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:59:16.066279846Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:59:16.066310319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:59:16.071975754Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:59:18.635407231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:59:19.6220089Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-19T03:59:22.269663594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T03:59:22.275468705Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T03:59:23.423897432Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-19T03:59:23.635198187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:59:24.938238444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T03:59:24.942375156Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T03:59:26.857185803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:59:26.86367996Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:59:26.863702023Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:59:26.863730699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:59:26.869818803Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:59:27.226974359Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T03:59:27.307133733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T03:59:27.310213128Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T03:59:28.10344035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T03:59:28.108045054Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-19T03:59:28.635759896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:59:29.787257518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T03:59:29.789946125Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T03:59:31.029936545Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-19T03:59:32.370097124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T03:59:32.373422831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T03:59:33.419655882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:59:33.425889962Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:59:33.635891469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T03:59:34.832675061Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-19T03:59:38.51125185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T03:59:38.513984323Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T03:59:38.635661178Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T03:59:38.635662142Z caller=checkpoint.go:384 msg="flushing series" totalSize="581 kB" series=5
level=debug ts=2026-05-19T03:59:38.635686188Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T03:59:38.635682102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:59:38.635693474Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T03:59:38.639187636Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-19T03:59:38.640390844Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005417909s
ts=2026-05-19T03:59:39.424455709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T03:59:39.795406933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T03:59:39.797976859Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T03:59:40.627948934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T03:59:40.635314476Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T03:59:41.227253664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T03:59:42.581327099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T03:59:42.584067603Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-19T03:59:43.635139923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:59:44.721986169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T03:59:44.724708449Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T03:59:48.436399678Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T03:59:48.635956358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:59:48.930234327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T03:59:48.933310239Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-19T03:59:48.944651707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54614"
level=info ts=2026-05-19T03:59:50.342918542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T03:59:50.345683619Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T03:59:52.489978311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T03:59:52.498757695Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-19T03:59:53.635894518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T03:59:53.716245075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:59:53.720124004Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:59:53.720147113Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:59:53.72016985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T03:59:53.723103905Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T03:59:56.351604266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T03:59:56.354505172Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T03:59:56.516149507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:59:56.518930294Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:59:56.518949678Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T03:59:56.518971738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T03:59:56.520964771Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T03:59:57.736593245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T03:59:57.742801128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T03:59:58.635395005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:00:01.397624301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:00:01.400943265Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T04:00:02.413973494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:00:02.419597439Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T04:00:03.635350108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:00:06.392849962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:00:06.399483081Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:00:08.617575269Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:00:08.61758456Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-19T04:00:08.61884383Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-19T04:00:08.618886311Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976847 n=1
level=debug ts=2026-05-19T04:00:08.61892519Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976847 n=1
level=info ts=2026-05-19T04:00:08.635515318Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T04:00:08.635525437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:00:08.635557115Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:00:08.635566119Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:00:08.635572332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:00:08.635686757Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-19T04:00:08.63569744Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T04:00:08.635715027Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.727µs success=true
level=debug ts=2026-05-19T04:00:08.636313819Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=info ts=2026-05-19T04:00:08.682926156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:00:08.686386666Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:00:09.169701728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:00:09.172587813Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-19T04:00:09.42715367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T04:00:12.386239373Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-19T04:00:13.635744056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:00:13.772097438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:00:13.77497181Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:00:13.777102115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:00:13.779584881Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-19T04:00:14.022455248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46084"
level=info ts=2026-05-19T04:00:14.064039389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:00:14.068572044Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T04:00:16.140123647Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-19T04:00:16.140155158Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-05-19T04:00:16.639837251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:00:16.643515468Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T04:00:18.635850621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:00:18.947837003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48364"
level=debug ts=2026-05-19T04:00:19.88624915Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T04:00:20.684751287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:00:20.688077659Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:00:20.699092427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:00:20.701990313Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-19T04:00:20.702012267Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:00:20.702035243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:00:20.704158321Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:00:21.38409641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:00:21.387474422Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:00:22.024598261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:00:22.028410113Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T04:00:22.789417264Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T04:00:23.63489348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:00:23.636015455Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-19T04:00:27.160129534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:00:27.163110511Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T04:00:27.386432072Z caller=checkpoint.go:372 msg="writing series" size="361 kB"
level=warn ts=2026-05-19T04:00:28.635685851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:00:28.852820698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:00:28.856146763Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T04:00:29.951886939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T04:00:31.137130611Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-19T04:00:31.137154792Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-19T04:00:33.63535029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:00:34.483212511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:00:34.48628738Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T04:00:34.886582608Z caller=checkpoint.go:372 msg="writing series" size="646 kB"
level=info ts=2026-05-19T04:00:35.0037455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:00:35.006678073Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T04:00:35.93278621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:00:35.935746084Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:00:36.204310418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:00:36.207242111Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T04:00:38.635667296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:00:38.635652598Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:00:38.635687251Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:00:38.635699263Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:00:38.636915243Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
ts=2026-05-19T04:00:39.428213013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T04:00:42.088990237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:00:42.091821049Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T04:00:42.386593645Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-19T04:00:42.739925491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:00:42.742922584Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:00:42.742948777Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:00:42.742974537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:00:42.745345601Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=warn ts=2026-05-19T04:00:43.635628082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:00:44.864589237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:00:44.868056066Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:00:44.868076923Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:00:44.868101949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:00:44.870423612Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T04:00:46.042303075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:00:46.045310983Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-19T04:00:46.13664835Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-19T04:00:48.63560991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:00:48.95188673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59776"
level=debug ts=2026-05-19T04:00:49.886437949Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-19T04:00:50.206813113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:00:50.213790379Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T04:00:51.361381013Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:00:52.324921453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:00:52.327550603Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T04:00:53.635220815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:00:53.636455163Z caller=checkpoint.go:372 msg="writing series" size="342 kB"
level=debug ts=2026-05-19T04:00:53.636469029Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-19T04:00:54.515776502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:00:54.519431196Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:00:56.082165853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:00:56.087414773Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:00:56.087438607Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:00:56.087464947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:00:56.092518674Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:00:57.38686779Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-19T04:00:58.635376683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:00:59.556828649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:00:59.561339451Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T04:01:01.15683987Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-19T04:01:01.156870473Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=2
level=info ts=2026-05-19T04:01:01.826893679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:01:01.833276659Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:01:01.885360566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:01:01.888077803Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=warn ts=2026-05-19T04:01:03.634892672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:01:04.886063342Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
level=debug ts=2026-05-19T04:01:05.264442714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T04:01:05.267076614Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T04:01:06.460581156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="629 B" avg_comp="629 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:01:06.46309009Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-19T04:01:06.463110453Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:06.463130803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:01:06.465045281Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:01:08.617477463Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:01:08.617484114Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:01:08.63544925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:01:08.635474688Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:01:08.635484777Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:01:08.635496991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:01:08.636718238Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-05-19T04:01:09.419364105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:01:09.422702509Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:01:09.422722295Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:09.422742641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:01:09.424871205Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-19T04:01:09.430196007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T04:01:09.868074363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:01:09.873228417Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T04:01:10.699554393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:01:10.70252373Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:01:11.512808055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:01:11.515571223Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:01:11.515591712Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:11.515612305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:01:11.517894312Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T04:01:12.386290839Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-19T04:01:13.628584543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:01:13.632854459Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:01:13.632876057Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:13.632896144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-19T04:01:13.634879289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-19T04:01:13.635085957Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:01:13.722344151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:01:13.726676602Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:01:15.630089706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:01:15.633506683Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T04:01:16.137027931Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=info ts=2026-05-19T04:01:17.342154551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:01:17.344861546Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:01:18.259749235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:01:18.262831158Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:01:18.262853019Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:18.262879516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:01:18.265500544Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:01:18.635680619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:01:19.314280754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:01:19.31689061Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T04:01:19.886252128Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-19T04:01:22.204876794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:01:22.208282972Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:01:22.208315822Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:22.208337838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:01:22.211682739Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:01:22.894484011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:01:22.897374813Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:01:22.897403926Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:22.897429818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:01:22.89993159Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-19T04:01:23.635303262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:01:23.636459254Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T04:01:23.766593162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:01:23.769577058Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T04:01:23.987805703Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:01:24.465217869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:01:24.468057568Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T04:01:27.386364387Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-19T04:01:27.386393686Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-19T04:01:27.679469977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:01:27.682494349Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:01:27.780748032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:01:27.783648817Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T04:01:28.230983943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:01:28.233972336Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:01:28.233996818Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:28.23402485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:01:28.236448277Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T04:01:28.635549355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:01:29.877381152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:01:29.881520171Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-19T04:01:31.136218499Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-19T04:01:32.010817027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:01:32.014011467Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:01:33.408722148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:01:33.412389468Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:01:33.635829353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:01:34.390026961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:01:34.395526136Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T04:01:34.409665828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:01:34.413070021Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T04:01:34.886176272Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-19T04:01:36.069979477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:01:36.072686329Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-19T04:01:37.251850025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:38146"
level=info ts=2026-05-19T04:01:37.959275136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:01:37.966298693Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:01:38.635109463Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:01:38.635139918Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:01:38.635148718Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:01:38.635166286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:01:38.636280637Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-19T04:01:39.432503151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T04:01:40.20648587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:01:40.20928129Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:01:40.449070595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:01:40.451780995Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T04:01:40.593063123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:01:40.595655762Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:01:40.595674557Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:40.595696287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:01:40.597899048Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T04:01:42.386184739Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-19T04:01:43.628589124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:01:43.631418116Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:01:43.631437917Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:43.631460148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:01:43.634786121Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T04:01:43.6358022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:01:46.136479774Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-19T04:01:48.300344688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:01:48.303336629Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:01:48.353778758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:01:48.360584039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T04:01:48.635411781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:01:48.958812698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56156"
level=info ts=2026-05-19T04:01:49.330900686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:01:49.333991282Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T04:01:49.886658906Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-05-19T04:01:50.887054908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:01:50.890644166Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T04:01:51.811502673Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:01:53.545571193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:01:53.54868401Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-19T04:01:53.635360959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:01:53.636447929Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-19T04:01:54.888281019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:01:54.891464688Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T04:01:56.394838524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T04:01:56.493342726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:01:56.497522192Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:01:56.497568903Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:01:56.497600616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:01:56.501225688Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:01:57.021372381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:01:57.024159478Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T04:01:57.386926262Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-19T04:01:58.009791918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:01:58.012429472Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T04:01:58.635021363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:02:01.137152544Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T04:02:01.637424686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:02:01.642263094Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T04:02:01.737420247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:02:01.740041455Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:02:02.077911376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:02:02.08067934Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=warn ts=2026-05-19T04:02:03.635693004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:02:03.77393837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:02:03.776437486Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:02:04.886711229Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-19T04:02:05.448397967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:02:05.451159168Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T04:02:08.617098556Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:02:08.61814348Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T04:02:08.635024177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:02:08.635052391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:02:08.635060429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:02:08.635134368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:02:08.63631359Z caller=checkpoint.go:372 msg="writing series" size="275 kB"
level=info ts=2026-05-19T04:02:08.687467345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:02:08.690974018Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-19T04:02:09.433772911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T04:02:11.744563849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:02:11.748453179Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:02:11.748475831Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:02:11.748497496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:02:11.752055162Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T04:02:12.024903075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:02:12.029600749Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T04:02:12.386371924Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-19T04:02:13.635240332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:02:14.027646448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51832"
level=debug ts=2026-05-19T04:02:16.136918095Z caller=checkpoint.go:372 msg="writing series" size="588 kB"
level=debug ts=2026-05-19T04:02:16.13693916Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=13
level=warn ts=2026-05-19T04:02:18.635402798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:02:18.962576752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46056"
level=debug ts=2026-05-19T04:02:19.887154826Z caller=checkpoint.go:372 msg="writing series" size="684 kB"
level=info ts=2026-05-19T04:02:22.810405188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:02:22.814937275Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:02:22.903069691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:02:22.9064751Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T04:02:23.116173826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:02:23.121704561Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T04:02:23.635283893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:02:23.636518802Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=info ts=2026-05-19T04:02:23.719708253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:02:23.722647171Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T04:02:23.903689701Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T04:02:27.386242421Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-19T04:02:28.141514395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:02:28.144398738Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-19T04:02:28.635642639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:02:29.093570565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:02:29.096634211Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=debug ts=2026-05-19T04:02:31.136772676Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-19T04:02:31.13679431Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-19T04:02:33.115217329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:02:33.119295969Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T04:02:33.458572566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T04:02:33.635563275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:02:34.88660006Z caller=checkpoint.go:372 msg="writing series" size="774 kB"
level=info ts=2026-05-19T04:02:35.984280146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:02:35.990414517Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:02:38.635565628Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T04:02:38.635558341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:02:38.63558777Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:02:38.635596178Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:02:38.636671186Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-19T04:02:38.913583597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:02:38.916411362Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-19T04:02:39.43616321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T04:02:39.913124779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:02:39.91559806Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:02:40.064953639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:02:40.071443658Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T04:02:42.386666988Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-19T04:02:43.635568931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:02:43.855188475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:02:43.857947942Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:02:45.163594681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:02:45.166402895Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T04:02:46.136327119Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-19T04:02:46.784056032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:02:46.786694748Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T04:02:48.635775948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:02:48.901987936Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T04:02:49.886265889Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-19T04:02:49.918547074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:02:49.923193198Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:02:49.956164223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:02:49.958530544Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T04:02:50.008641475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:02:50.011612879Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:02:52.49648195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:02:52.500038825Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:02:53.635800072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:02:53.636989832Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=debug ts=2026-05-19T04:02:53.6370049Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-19T04:02:57.256936092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:02:57.260036648Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T04:02:57.386357035Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T04:02:58.009210496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:02:58.011945376Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:02:58.233552444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:02:58.237198185Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T04:02:58.635435803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:02:59.847021077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:02:59.849693233Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T04:03:01.13671587Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-19T04:03:03.635159377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:03:04.886590297Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-19T04:03:06.009720179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:03:06.012217841Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T04:03:08.617323956Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:03:08.617794005Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-19T04:03:08.635822728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:03:08.635832746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:03:08.635848024Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:03:08.635856062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:03:08.635951733Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-19T04:03:09.438285997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T04:03:10.112519837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:03:10.115023851Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-19T04:03:12.386312265Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-19T04:03:13.635456776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:03:14.032075277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50408"
level=debug ts=2026-05-19T04:03:14.640406086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T04:03:15.055984028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:03:15.361739943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:03:15.366756702Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:03:16.136972539Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=warn ts=2026-05-19T04:03:18.635638947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:03:18.969987383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42926"
level=debug ts=2026-05-19T04:03:19.886756644Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-19T04:03:20.337117424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:03:20.341895803Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:03:22.735455768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:03:22.739516419Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:03:23.54658071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:03:23.549187037Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T04:03:23.635458457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:03:23.636736993Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-19T04:03:25.781926688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:03:25.784350785Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:03:26.961719969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:03:26.964090699Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T04:03:27.388231084Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-19T04:03:27.388252701Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=9
level=warn ts=2026-05-19T04:03:28.635401707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:03:28.933011353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:03:28.938961193Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T04:03:31.136818113Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T04:03:33.298901337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:03:33.302015036Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:03:33.635574715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:03:34.445158491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:03:34.451427893Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T04:03:34.886822624Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-19T04:03:36.429063264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:03:36.432424304Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:03:37.054070289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:03:37.056659303Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T04:03:38.005168911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:03:38.007587578Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:03:38.635476826Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T04:03:38.635478486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:03:38.635498246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:03:38.63550691Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:03:38.636770031Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=info ts=2026-05-19T04:03:38.961491328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:03:38.963774757Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:03:39.001902505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:03:39.004350501Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-19T04:03:39.441114279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:03:40.070477552Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T04:03:42.386907971Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=warn ts=2026-05-19T04:03:43.635116194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:03:44.032922331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35808"
level=info ts=2026-05-19T04:03:44.668071313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:03:44.671189402Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:03:45.702562859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:03:45.705270336Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:03:46.137044936Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T04:03:46.13706478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-19T04:03:48.525631981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:03:48.527974565Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T04:03:48.635215572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:03:48.973274021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40080"
level=debug ts=2026-05-19T04:03:49.886611159Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-19T04:03:50.273732985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:03:50.276124505Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:03:50.800580908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:03:50.80821159Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:03:51.390754876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:03:51.394102645Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T04:03:53.635070081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:03:53.653147774Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-19T04:03:53.653166314Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=2
level=info ts=2026-05-19T04:03:56.054123205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:03:56.056978143Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T04:03:57.386607278Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=warn ts=2026-05-19T04:03:58.635655876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:04:00.852667685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T04:04:01.136599521Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=warn ts=2026-05-19T04:04:03.635710888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:04:04.459726908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:04:04.46238587Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T04:04:04.886958104Z caller=checkpoint.go:372 msg="writing series" size="466 kB"
level=info ts=2026-05-19T04:04:05.257852278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:04:05.262328702Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:04:06.527823839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:04:08.617386709Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:04:08.617807395Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:04:08.635563696Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:04:08.635581522Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:04:08.635589328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:04:08.635589185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:04:08.636701602Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
ts=2026-05-19T04:04:09.443709506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:04:09.981530493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:04:09.983757721Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=debug ts=2026-05-19T04:04:12.38698315Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=debug ts=2026-05-19T04:04:12.387005381Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-19T04:04:13.635134888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:04:14.033624488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49332"
level=info ts=2026-05-19T04:04:15.831002998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:04:15.83534966Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:04:15.858414855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:04:15.860589113Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T04:04:16.136461109Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-19T04:04:18.635451575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:04:18.976228262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45942"
level=debug ts=2026-05-19T04:04:19.886247183Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-19T04:04:20.583020173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:04:20.589655543Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:04:21.114407744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:04:21.117364882Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T04:04:21.318899891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:04:21.321515682Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:04:22.970998698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:04:22.973502658Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T04:04:23.635831835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:04:23.635894248Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T04:04:23.692040162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:04:23.694447255Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:04:23.983966321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:04:23.986198473Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T04:04:27.386931337Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-19T04:04:28.471886204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:04:28.479662692Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T04:04:28.63492638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:04:28.897086788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:04:28.899423137Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T04:04:31.136161812Z caller=checkpoint.go:372 msg="writing series" size="9.8 kB"
level=info ts=2026-05-19T04:04:32.298294176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:04:32.30117441Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T04:04:32.751550238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:04:32.754175814Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:04:32.870513978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:04:32.873446466Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:04:33.635576494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:04:34.886358488Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-19T04:04:36.192579303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:04:36.195284239Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T04:04:36.469200851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:04:36.864375074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:04:36.866949504Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:04:38.634994733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:04:38.635019992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:04:38.635028573Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:04:38.635089508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:04:38.636111955Z caller=checkpoint.go:384 msg="flushing series" totalSize="928 kB" series=6
level=info ts=2026-05-19T04:04:38.640746471Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-19T04:04:38.641604405Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005745904s
ts=2026-05-19T04:04:39.445793956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T04:04:39.68418582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:04:39.690094414Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:04:39.885642868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:04:39.88905044Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T04:04:43.635887425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:04:43.693059384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:04:43.696094766Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-19T04:04:44.03465956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34292"
level=warn ts=2026-05-19T04:04:48.63513489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:04:48.979425119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41870"
level=info ts=2026-05-19T04:04:49.050136877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:04:49.059814962Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:04:50.65850193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T04:04:50.660257737Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T04:04:53.635831735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:04:56.162199912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:04:56.166186262Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=warn ts=2026-05-19T04:04:58.634900972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:05:01.860929713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:05:01.863840772Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T04:05:03.635785867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:05:04.904386135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:05:04.909976366Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-19T04:05:07.271391076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:36558"
level=info ts=2026-05-19T04:05:07.430289985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:05:07.43327919Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:05:08.61747388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:05:08.617875584Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:05:08.634969682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:05:08.634982873Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:05:08.634990693Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:05:08.634987472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:05:08.634987761Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T04:05:08.635104532Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-19T04:05:08.635113185Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T04:05:08.635129395Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12µs success=true
level=debug ts=2026-05-19T04:05:08.635765371Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
ts=2026-05-19T04:05:09.44760905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T04:05:10.083048771Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:05:11.305176826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:05:11.307718054Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:05:11.307737832Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:05:11.307757601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:05:11.309821562Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T04:05:12.3855883Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-19T04:05:12.472989836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:05:12.475384721Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-19T04:05:13.63540726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:05:15.049717875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:05:15.052052478Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T04:05:16.136529926Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-19T04:05:16.136571037Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-05-19T04:05:16.540957194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:05:16.544124863Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:05:18.635643857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:05:18.971208625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:05:18.977772724Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-19T04:05:18.982482304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:32792"
level=info ts=2026-05-19T04:05:19.375804741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:05:19.378265518Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:05:19.60683367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:05:19.609758563Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T04:05:19.885593459Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T04:05:20.762185475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:05:20.764788301Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:05:21.353294954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:05:21.355762617Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:05:21.599892641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:05:21.602053918Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:05:23.32071854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:05:23.323084644Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T04:05:23.635010763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:05:23.63614217Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-19T04:05:24.195925472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:05:24.201514977Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T04:05:26.55596019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T04:05:27.38639054Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=warn ts=2026-05-19T04:05:28.635125287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:05:28.780514956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:05:28.783275935Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T04:05:29.625530809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:05:29.627937998Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:05:30.391030698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:05:30.393521378Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T04:05:31.136394395Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-19T04:05:31.13641603Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-19T04:05:31.905903991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:05:31.90844221Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-19T04:05:33.635272837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:05:34.886450506Z caller=checkpoint.go:372 msg="writing series" size="647 kB"
level=info ts=2026-05-19T04:05:35.958410309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:05:35.962042465Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T04:05:38.635697351Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:05:38.63572876Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:05:38.635742129Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:05:38.635745407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:05:38.635816019Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
ts=2026-05-19T04:05:39.449387603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:05:42.324150358Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T04:05:42.386205937Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-19T04:05:43.634885864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:05:45.031707138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:05:45.03508197Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=debug ts=2026-05-19T04:05:46.135970197Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-19T04:05:47.112786547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:05:47.117679953Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T04:05:48.63564397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:05:49.8855866Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-19T04:05:50.811226439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:05:50.814792838Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:05:51.153557016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:05:51.155782351Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T04:05:53.397741888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:05:53.400281158Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=warn ts=2026-05-19T04:05:53.635888384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:05:53.636015073Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-19T04:05:53.636028755Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-19T04:05:56.489376766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:05:56.492179286Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T04:05:57.38575498Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-19T04:05:57.552285963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:05:57.556893424Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:05:58.635551893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:06:01.152381843Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-19T04:06:01.152404343Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=2
level=warn ts=2026-05-19T04:06:03.635209612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:06:04.885359412Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
level=info ts=2026-05-19T04:06:05.677895763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:06:05.680857177Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:06:05.934496935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:06:05.937121752Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:06:08.61734309Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:06:08.617828825Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:06:08.635104609Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:06:08.635119394Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:06:08.635126164Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:06:08.635133391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:06:08.636436176Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-19T04:06:09.317572656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:06:09.320038434Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-19T04:06:09.452618585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:06:09.825384766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:06:10.584566515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:06:10.587051539Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:06:10.587069095Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:06:10.5870892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:06:10.589109664Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T04:06:12.386037602Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-19T04:06:12.686123001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="629 B" avg_comp="629 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:06:12.688567203Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T04:06:13.635368804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:06:13.90519966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:06:13.911181349Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:06:13.911201628Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:06:13.911226145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:06:13.916783738Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:06:13.961861932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:06:13.96741303Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:06:13.96742682Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
ts=2026-05-19T04:06:14.041386861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36144"
level=info ts=2026-05-19T04:06:14.211247611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:06:14.217011367Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:06:16.042107189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:06:16.046128141Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T04:06:16.136553606Z caller=checkpoint.go:372 msg="writing series" size="488 kB"
level=info ts=2026-05-19T04:06:17.509065191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:06:17.511608226Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T04:06:18.345933073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:06:18.348440887Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-19T04:06:18.635109858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:06:19.886185618Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-19T04:06:21.542791426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:06:21.545309168Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:06:21.788924453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T04:06:22.371529728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:06:22.375274252Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T04:06:23.635802135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:06:23.635852657Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T04:06:24.766088887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:06:24.768842422Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:06:24.768861349Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:06:24.768881259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:06:24.770967806Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:06:25.931582472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:06:25.937580323Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T04:06:27.386279956Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-19T04:06:27.386302678Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-19T04:06:27.641317413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:06:27.644166591Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T04:06:28.635651052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:06:28.94085681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:06:28.943561672Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:06:29.756703597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:06:29.762729438Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T04:06:31.136406997Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-19T04:06:33.635727681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:06:34.885742677Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-19T04:06:35.023227727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:06:35.026081279Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:06:38.634988964Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T04:06:38.634989741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:06:38.635012323Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:06:38.635020915Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:06:38.636124839Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T04:06:39.206173716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:06:39.210936979Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-19T04:06:39.454914071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:06:40.818550961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:06:40.821307469Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:06:42.300105083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:06:42.302348051Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T04:06:42.385602957Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-19T04:06:42.762767019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:06:42.765325042Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T04:06:43.635269878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:06:43.64249689Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:06:45.608730053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:06:45.613492333Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T04:06:46.135936947Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-19T04:06:46.734679246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:06:46.737816104Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:06:48.28796266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:06:48.290420114Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-19T04:06:48.635415779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:06:48.992244721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39524"
level=info ts=2026-05-19T04:06:49.489749447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:06:49.492147668Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-19T04:06:49.492166058Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:06:49.492187444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:06:49.49421334Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T04:06:49.886313786Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-19T04:06:50.101975836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:06:50.110650915Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:06:50.602770761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:06:50.60580397Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:06:53.635385961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:06:53.63540231Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-19T04:06:53.928455976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:06:53.931057178Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T04:06:57.386291721Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-19T04:06:58.298465378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:06:58.302423515Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:06:58.305458882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:06:58.307702002Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-19T04:06:58.635664349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:07:01.136295146Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T04:07:01.498085922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:07:01.500574705Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:07:01.796352395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:07:01.803333878Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:07:03.308483808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:07:03.310809373Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T04:07:03.635492363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:07:04.342476093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:07:04.344720451Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T04:07:04.820239505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:07:04.822693594Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T04:07:04.885926932Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-19T04:07:05.679194966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:07:07.278350813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60596"
level=info ts=2026-05-19T04:07:08.549354357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:07:08.551997378Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T04:07:08.61714791Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:07:08.618202563Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T04:07:08.634932627Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T04:07:08.634933757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:07:08.634948243Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:07:08.634957679Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:07:08.636191909Z caller=checkpoint.go:372 msg="writing series" size="275 kB"
level=info ts=2026-05-19T04:07:08.706395335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:07:08.709729522Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-19T04:07:09.456131126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T04:07:12.386027928Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-19T04:07:13.635160992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:07:14.043738257Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:07:14.791415398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:07:14.794152253Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T04:07:16.136271673Z caller=checkpoint.go:372 msg="writing series" size="476 kB"
level=debug ts=2026-05-19T04:07:16.136294757Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=13
level=warn ts=2026-05-19T04:07:18.635102091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:07:19.499277358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:07:19.505780364Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:07:19.505799156Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:07:19.505819003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:07:19.510211367Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:07:19.886397487Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=warn ts=2026-05-19T04:07:23.635586781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:07:23.635699069Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=info ts=2026-05-19T04:07:25.31482626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:07:25.3184142Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:07:25.318436365Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:07:25.318456698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:07:25.320507749Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T04:07:27.386156701Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=warn ts=2026-05-19T04:07:28.635491857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:07:30.323207387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:07:30.32677586Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T04:07:31.135782377Z caller=checkpoint.go:372 msg="writing series" size="287 kB"
level=debug ts=2026-05-19T04:07:31.135805008Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-19T04:07:31.195722374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:07:31.198360926Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:07:32.477951642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:07:32.480675645Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:07:33.272929841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:07:33.279491875Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T04:07:33.635317852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:07:34.88635892Z caller=checkpoint.go:372 msg="writing series" size="729 kB"
level=info ts=2026-05-19T04:07:35.616372433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:07:35.621175512Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T04:07:37.371450767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:07:37.37389219Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:07:37.373909202Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:07:37.373929092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:07:37.376239922Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:07:38.635811524Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:07:38.635833933Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:07:38.635842686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:07:38.635833713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:07:38.635840017Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=debug ts=2026-05-19T04:07:39.116295405Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:07:39.366028493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:07:39.36998661Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-19T04:07:39.457452138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T04:07:40.939157679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:07:40.941813695Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:07:42.09382185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:07:42.096241268Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T04:07:42.38604249Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-19T04:07:42.483342591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:07:42.485506775Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-19T04:07:42.485521081Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:07:42.485556323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:07:42.487339411Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T04:07:43.635840386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:07:44.21084772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:07:44.212936578Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:07:44.537773115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:07:44.539821332Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:07:44.539835919Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:07:44.539857869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:07:44.541692804Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:07:45.384005896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:07:45.386393666Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:07:45.893086004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:07:45.896018663Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:07:45.896037277Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:07:45.896057706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:07:45.898414758Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T04:07:46.136210254Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-19T04:07:46.441869923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T04:07:47.441572999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:07:47.443798417Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:07:47.739570359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:07:47.745373061Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T04:07:48.63554549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:07:49.886265349Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-19T04:07:50.34879561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:07:50.353589016Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:07:50.353609136Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:07:50.353631085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:07:50.355892951Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:07:50.687015991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:07:50.689397465Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:07:50.689413654Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:07:50.689433439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:07:50.691447174Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:07:53.393779399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:07:53.39644513Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T04:07:53.635209425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:07:53.636448599Z caller=checkpoint.go:372 msg="writing series" size="311 kB"
level=debug ts=2026-05-19T04:07:53.636462406Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-19T04:07:55.301936855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:07:55.305751456Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T04:07:57.3858965Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T04:07:57.904369726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:07:57.909593501Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T04:07:58.634982034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:07:59.308374651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:07:59.31090143Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:07:59.310917385Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:07:59.31093424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:07:59.312912631Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T04:07:59.826510823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:07:59.829491849Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T04:08:00.457272532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:08:00.460959001Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:08:00.460977444Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:08:00.46099634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:08:00.462909201Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T04:08:01.135745898Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-19T04:08:02.885659358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:08:02.888295979Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-19T04:08:03.634917197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:08:04.219401523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T04:08:04.885993962Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-19T04:08:05.116727144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:08:05.119641118Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:08:05.119659736Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:08:05.119681523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:08:05.121768819Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:08:07.21616164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:08:07.218798924Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:08:07.439378112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:08:07.441994415Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T04:08:08.617297265Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:08:08.617688788Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:08:08.634898085Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:08:08.634912054Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:08:08.634919562Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:08:08.63494102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:08:08.63604521Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-19T04:08:09.458811243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:08:10.69157763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:08:10.694707194Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T04:08:12.386230431Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-05-19T04:08:13.234577988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:08:13.237158943Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T04:08:13.635341128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:08:16.136024756Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-19T04:08:16.199146884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:08:16.201602025Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:08:16.201619225Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:08:16.201637754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:08:16.20402922Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T04:08:17.790294993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:08:17.79306987Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:08:18.312489948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:08:18.315280225Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T04:08:18.635697896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:08:19.88560124Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-19T04:08:20.578308368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:08:20.580735453Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:08:23.46986554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:08:23.473382474Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:08:23.473401154Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:08:23.473419871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:08:23.475228678Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:08:23.56144944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:08:23.564766885Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:08:23.564779574Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=warn ts=2026-05-19T04:08:23.635149803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:08:23.636499607Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-19T04:08:23.729561089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:08:23.731622258Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:08:23.773698557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:08:23.7769107Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:08:24.21968513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:08:24.225274109Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:08:25.440717504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:08:25.443040273Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-19T04:08:27.387137106Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-19T04:08:27.387159626Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=9
level=info ts=2026-05-19T04:08:28.408660975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:08:28.411485006Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T04:08:28.635843572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:08:28.848224217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:08:28.850737117Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:08:29.752964874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:08:29.755761128Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:08:30.644389101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:08:30.647038392Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T04:08:31.136233565Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T04:08:32.805833838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:08:32.808219848Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T04:08:33.095060603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:08:33.097343063Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-19T04:08:33.635768682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:08:34.733571876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:08:34.736347633Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:08:34.886010315Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-19T04:08:35.933168385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:08:35.937218851Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:08:36.746567531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:08:36.749170354Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:08:37.302581712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:08:37.305183266Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:08:38.035285895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:08:38.038917756Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:08:38.038937088Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:08:38.038958436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:08:38.042063012Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T04:08:38.23365965Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:08:38.634996768Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T04:08:38.635008535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:08:38.635019771Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:08:38.635028123Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:08:38.636226562Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=info ts=2026-05-19T04:08:39.201010938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:08:39.203889054Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-19T04:08:39.460994411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:08:42.386160618Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=warn ts=2026-05-19T04:08:43.635878446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:08:43.705424819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T04:08:43.706930731Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T04:08:44.051505539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37508"
level=debug ts=2026-05-19T04:08:46.136378913Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T04:08:46.136400745Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-19T04:08:47.3492504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:08:47.355490734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:08:48.068576771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:08:48.073514983Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T04:08:48.635660706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:08:49.886114567Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-19T04:08:51.08574062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:08:51.088975305Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-19T04:08:53.635732774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:08:53.646690935Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-19T04:08:53.646707954Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=2
level=info ts=2026-05-19T04:08:54.450186553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:08:54.454187032Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T04:08:57.155045477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:08:57.157601336Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T04:08:57.386252427Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=warn ts=2026-05-19T04:08:58.635513387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:09:01.136342255Z caller=checkpoint.go:372 msg="writing series" size="275 kB"
level=info ts=2026-05-19T04:09:03.133578753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:09:03.137951724Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T04:09:03.635863006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:09:04.000175294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:09:04.004152878Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T04:09:04.886450463Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=debug ts=2026-05-19T04:09:04.886469326Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-19T04:09:04.967221831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:09:04.971301773Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:09:05.005302484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:09:05.007592464Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T04:09:06.749453928Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:09:07.986760654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:09:07.990216163Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:09:08.617288767Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:09:08.617714995Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:09:08.635131689Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:09:08.635149186Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:09:08.6351583Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:09:08.635162975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:09:08.636273323Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
ts=2026-05-19T04:09:09.464614326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T04:09:09.735417385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:09:09.738367512Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:09:10.678719113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:09:10.681167944Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:09:11.612643632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:09:11.618688166Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-19T04:09:12.385622676Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-19T04:09:13.635805497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:09:14.052900301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34862"
level=debug ts=2026-05-19T04:09:16.13639417Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-19T04:09:17.552931414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:09:17.555723271Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T04:09:18.635441414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:09:19.008213914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37022"
level=info ts=2026-05-19T04:09:19.016098887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:09:19.018462596Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T04:09:19.886332664Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-19T04:09:21.071660883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:09:21.073978592Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=warn ts=2026-05-19T04:09:23.6354232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:09:23.635465026Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T04:09:24.479732736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:09:24.482299141Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:09:25.858362382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:09:25.861367267Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T04:09:26.265335685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:09:26.267909089Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:09:26.425379911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:09:26.427885029Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T04:09:27.386264773Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-19T04:09:28.067202132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:09:28.070812268Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T04:09:28.635594182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:09:30.773810224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:09:30.778755766Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:09:31.13583335Z caller=checkpoint.go:372 msg="writing series" size="9.8 kB"
level=info ts=2026-05-19T04:09:32.33844749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:09:32.341113341Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T04:09:33.635256771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:09:34.885831011Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-19T04:09:35.520495358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:09:35.52314734Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-19T04:09:38.635714451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:09:38.635716759Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=8
level=info ts=2026-05-19T04:09:38.635741584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:09:38.635758589Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:09:38.635766575Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T04:09:38.63944911Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-19T04:09:38.64029381Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005032997s
ts=2026-05-19T04:09:39.467667981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:09:40.177778087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T04:09:41.510670255Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:09:43.265881408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:09:43.271923937Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T04:09:43.635497407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:09:43.884035915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:09:43.886319745Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T04:09:44.179798384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:09:44.181947878Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T04:09:48.634901235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:09:52.082580636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:09:52.093364667Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:09:52.221858044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:09:52.224293946Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T04:09:53.635193311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-19T04:09:58.634872848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:10:00.075658058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:10:00.079772889Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:10:01.254306647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:10:01.257038483Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:10:02.072382159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:10:02.075483184Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:10:03.309055223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:10:03.311673497Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T04:10:03.63572679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:10:08.617331738Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:10:08.617754614Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:10:08.634947402Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:10:08.634962669Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:10:08.634971171Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-19T04:10:08.634979124Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-19T04:10:08.634997235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:10:08.635107768Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-19T04:10:08.635116571Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T04:10:08.635132937Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.954µs success=true
level=debug ts=2026-05-19T04:10:08.635716945Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-19T04:10:08.635726116Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-19T04:10:08.673306423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:10:08.677362891Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:10:09.096887695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:10:09.099495121Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
ts=2026-05-19T04:10:09.469405784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T04:10:11.020365805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:10:11.022918553Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:10:11.205238597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:10:11.20767241Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:10:11.255955295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:10:11.261516984Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:10:12.102923639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:10:12.105651594Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T04:10:12.385322063Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-19T04:10:13.394175336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:10:13.396651535Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=warn ts=2026-05-19T04:10:13.635388961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:10:14.055486717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49456"
level=debug ts=2026-05-19T04:10:14.309909815Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:10:14.671909031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:10:14.677433938Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:10:16.136166472Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-19T04:10:16.136188621Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
level=warn ts=2026-05-19T04:10:18.635653207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:10:18.64669102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T04:10:19.655274855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:10:19.65844629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T04:10:19.885963631Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-19T04:10:22.150813037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:10:22.154284853Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:10:23.635597816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:10:23.63560617Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-19T04:10:25.317798035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:10:25.324126808Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:10:26.523571389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:10:26.526110767Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:10:27.386345696Z caller=checkpoint.go:372 msg="writing series" size="383 kB"
level=warn ts=2026-05-19T04:10:28.635801431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:10:28.844350119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:10:28.846764998Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T04:10:31.136931862Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-19T04:10:31.136952923Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-19T04:10:33.635718246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:10:34.885715016Z caller=checkpoint.go:372 msg="writing series" size="647 kB"
level=info ts=2026-05-19T04:10:35.436131919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:10:35.438597523Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:10:38.635209544Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T04:10:38.63521014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:10:38.635230998Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:10:38.635241244Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:10:38.635417734Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
ts=2026-05-19T04:10:39.470590651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T04:10:40.65555282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:10:40.660367383Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T04:10:42.386085036Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-19T04:10:43.635578438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:10:43.64371832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:10:43.646411223Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T04:10:44.780872768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:10:44.783438394Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:10:46.095450963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:10:46.099753112Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T04:10:46.135998667Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-19T04:10:47.823025665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:10:47.825379629Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-19T04:10:48.635736939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:10:48.970079408Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T04:10:49.88535639Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-19T04:10:50.626525513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:10:50.628979379Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-19T04:10:53.635371988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:10:53.635486931Z caller=checkpoint.go:372 msg="writing series" size="380 kB"
level=debug ts=2026-05-19T04:10:53.63550432Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-19T04:10:54.869906149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:10:54.876377186Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:10:55.755694263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="629 B" avg_comp="629 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:10:55.758055283Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T04:10:57.386273177Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-19T04:10:57.801293123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:10:57.803431035Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T04:10:58.635805655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:11:00.453922315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:11:00.456476611Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T04:11:01.145828478Z caller=checkpoint.go:372 msg="writing series" size="19 MB"
level=debug ts=2026-05-19T04:11:01.14585588Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=2
level=warn ts=2026-05-19T04:11:03.635570708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:11:04.744184211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:11:04.746401254Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T04:11:04.88606797Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
level=info ts=2026-05-19T04:11:04.923341918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:11:04.925478616Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:11:06.48573447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:11:06.490758086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:11:08.328161141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T04:11:08.616979746Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:11:08.618029612Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T04:11:08.635847588Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:11:08.635863515Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T04:11:08.635858615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:11:08.635870655Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:11:08.636019359Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=info ts=2026-05-19T04:11:09.076050017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:11:09.078994465Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T04:11:09.34225638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:11:09.344679921Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
ts=2026-05-19T04:11:09.472200665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:11:09.560030316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:11:09.56239427Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:11:10.875385069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:11:10.881445587Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T04:11:11.369732592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:11:11.375644161Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T04:11:12.385558206Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-19T04:11:13.635482966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:11:13.975357798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:11:13.977712251Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-19T04:11:14.059717851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56480"
level=info ts=2026-05-19T04:11:15.908730472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:11:15.911462095Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:11:16.136421623Z caller=checkpoint.go:372 msg="writing series" size="488 kB"
level=warn ts=2026-05-19T04:11:18.635432452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:11:19.885594225Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-19T04:11:21.639800696Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T04:11:23.63568705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:11:23.635766334Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-19T04:11:23.885560569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:11:23.888246494Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T04:11:27.386307474Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-19T04:11:27.386328756Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-19T04:11:28.635166466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:11:28.886900276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T04:11:28.891746019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:11:28.892011053Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:11:28.894121909Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T04:11:31.135999056Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-19T04:11:33.635282225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:11:34.885918115Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-19T04:11:37.78450215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:11:37.787304096Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T04:11:38.635579697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:11:38.635594818Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:11:38.635616147Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:11:38.635624683Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:11:38.635619235Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-19T04:11:39.47492818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T04:11:40.002003982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:11:40.004515335Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:11:40.963199815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:11:40.969909653Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:11:41.576789785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:11:41.57925714Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T04:11:42.385606196Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-19T04:11:43.635281694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:11:45.901109769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:11:45.904916943Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T04:11:46.136008989Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-19T04:11:46.502937292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:11:46.505481286Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-19T04:11:48.635394751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:11:49.885782939Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-19T04:11:49.973012262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:11:49.975666925Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:11:50.522196072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:11:50.524682366Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:11:50.524700095Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:11:50.524719738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:11:50.526575205Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:11:50.538620616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:11:50.540458768Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:11:50.676772701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:11:50.678848528Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:11:52.49458047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:11:52.497407316Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:11:53.127250025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:11:53.132121268Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T04:11:53.635718719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:11:53.635767493Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-19T04:11:56.362898143Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:11:56.775056311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:11:56.777654556Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T04:11:57.385717628Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-19T04:11:58.635854456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:11:59.79574991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:11:59.797784524Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:12:01.136359176Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-19T04:12:01.457647406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T04:12:02.041711812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:12:02.045113952Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-19T04:12:03.635245041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:12:04.885958475Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-19T04:12:05.876728813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:12:05.883065049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:12:07.797205113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:12:07.800509038Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T04:12:08.25485933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:12:08.257522149Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:12:08.617515268Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:12:08.617933064Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:12:08.635024119Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:12:08.635041394Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:12:08.635050213Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:12:08.635044766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:12:08.636298268Z caller=checkpoint.go:372 msg="writing series" size="275 kB"
ts=2026-05-19T04:12:09.477450928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:12:10.509429003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:12:10.512187673Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:12:11.519126635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:12:11.524060788Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T04:12:12.386247887Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-19T04:12:13.635326205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:12:16.136140031Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
level=debug ts=2026-05-19T04:12:16.136162888Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=13
level=warn ts=2026-05-19T04:12:18.635014661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:12:18.889205203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:12:18.891868141Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-19T04:12:19.02638363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43988"
level=debug ts=2026-05-19T04:12:19.886390529Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=debug ts=2026-05-19T04:12:22.34415697Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:12:22.804298752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:12:22.807232688Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T04:12:23.635569008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:12:23.635715021Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=info ts=2026-05-19T04:12:23.642728149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:12:23.649224698Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:12:23.649238117Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:12:23.649262227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:12:23.655056956Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:12:24.630435674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:12:24.633054819Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:12:25.724185917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:12:25.72652544Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T04:12:27.385792126Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-19T04:12:27.556282554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:12:27.558509355Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T04:12:28.63511623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:12:28.769438561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:12:28.771769565Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T04:12:31.135890697Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
level=debug ts=2026-05-19T04:12:31.135911525Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-19T04:12:33.407500149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:12:33.410839915Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:12:33.410858496Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:12:33.410876413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:12:33.412941093Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-19T04:12:33.635562721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:12:34.886351951Z caller=checkpoint.go:372 msg="writing series" size="730 kB"
level=info ts=2026-05-19T04:12:35.625487689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:12:35.631830218Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-19T04:12:37.301509807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39610"
level=info ts=2026-05-19T04:12:38.634831775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:12:38.634857723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:12:38.634868025Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:12:38.634886797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:12:38.635968744Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-19T04:12:39.049754733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:12:39.05202333Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:12:39.052040482Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:12:39.052058964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:12:39.054683932Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:12:39.35569559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:12:39.359459039Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-19T04:12:39.480038155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T04:12:39.58475059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:12:39.589920376Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:12:42.385996565Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-19T04:12:43.136215463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:12:43.139039993Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=warn ts=2026-05-19T04:12:43.635162302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:12:45.785271845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:12:45.788022288Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:12:45.939564242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:12:45.945487228Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:12:45.945504724Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:12:45.945527182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:12:45.950943078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T04:12:46.135663552Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-19T04:12:47.056584316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:12:47.058974848Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:12:47.28152272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:12:47.283830576Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-19T04:12:48.635816184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:12:49.029461877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53378"
level=debug ts=2026-05-19T04:12:49.648778736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T04:12:49.886382439Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-19T04:12:49.948641792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:12:49.952056311Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T04:12:50.15044307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T04:12:50.152192444Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T04:12:50.353789612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:12:50.356307619Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:12:50.77367531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:12:50.776254908Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:12:52.795333203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:12:52.798300808Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-19T04:12:53.635348192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:12:53.635472519Z caller=checkpoint.go:372 msg="writing series" size="393 kB"
level=debug ts=2026-05-19T04:12:53.635487772Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-19T04:12:55.936084439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:12:55.939598712Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T04:12:57.386363157Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-19T04:12:58.635525085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:13:00.406186753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:13:00.41362757Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T04:13:01.135597073Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-19T04:13:01.851773885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:13:01.855387831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:13:03.635909761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:13:04.88607201Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-19T04:13:07.30326244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:46330"
level=info ts=2026-05-19T04:13:08.61729821Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:13:08.617296352Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:13:08.635736536Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:13:08.635754307Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:13:08.635763087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:13:08.635765318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:13:08.635778605Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-19T04:13:09.482839774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T04:13:11.382952323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:13:11.387646176Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T04:13:12.386355067Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-19T04:13:13.403196122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:13:13.40798953Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-19T04:13:13.635358716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:13:13.855569323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:13:13.858368105Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:13:14.557495423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:13:14.5639309Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:13:15.152729112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:13:15.155049002Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-19T04:13:16.135628207Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=debug ts=2026-05-19T04:13:16.210963783Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T04:13:18.634980569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:13:19.885672456Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-19T04:13:21.120061719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:13:21.122688188Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:13:22.091330262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:13:22.094116757Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-19T04:13:22.094133082Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:13:22.094152219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:13:22.096116076Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:13:22.713036706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:13:22.715576145Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:13:23.310147002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:13:23.312660053Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T04:13:23.635360351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:13:23.635598405Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-19T04:13:23.739644037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:13:23.744409087Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:13:23.744425998Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:13:23.744447257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:13:23.748738053Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T04:13:24.469337681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:13:24.471959175Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:13:25.197664065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:13:25.201849866Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T04:13:27.386808487Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-19T04:13:27.386829513Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=9
level=info ts=2026-05-19T04:13:28.334608533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:13:28.336982185Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:13:28.510484296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:13:28.517891136Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-19T04:13:28.634875234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:13:31.135825668Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-19T04:13:33.63560929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:13:34.885890842Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-19T04:13:35.088342268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:13:35.091443193Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:13:35.986758684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:13:35.99043486Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T04:13:36.889075468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:13:36.891902087Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:13:38.634841801Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:13:38.63486789Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T04:13:38.634871409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:13:38.634878437Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:13:38.636064533Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=debug ts=2026-05-19T04:13:39.226761383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:13:39.485568551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:13:40.146650327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:13:40.150606496Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:13:41.238562721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:13:41.24312246Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-19T04:13:42.144524567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:13:42.147497786Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:13:42.386260394Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-05-19T04:13:43.635524339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:13:46.136113179Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-19T04:13:46.136134454Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-19T04:13:46.29718994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:13:46.300498231Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:13:46.916464715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:13:46.919059752Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=debug ts=2026-05-19T04:13:47.613048695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T04:13:48.635781441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:13:49.885596848Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-19T04:13:53.635629762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:13:53.65066711Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-19T04:13:53.650684312Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=2
level=debug ts=2026-05-19T04:13:57.385616854Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=info ts=2026-05-19T04:13:57.722721378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:13:57.727283924Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:13:57.727301482Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:13:57.727322859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:13:57.729612104Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T04:13:58.144585654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:13:58.147600224Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:13:58.635831834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:14:00.376698468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:14:00.379804826Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T04:14:01.136188891Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=info ts=2026-05-19T04:14:02.024728102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:14:02.029318005Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T04:14:03.635584865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:14:04.74457072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:14:04.747039223Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T04:14:04.886642152Z caller=checkpoint.go:372 msg="writing series" size="604 kB"
level=debug ts=2026-05-19T04:14:04.88665689Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-19T04:14:05.952583231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:14:05.959021419Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:14:06.066367695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:14:06.06862947Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:14:06.272135342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:14:06.274601024Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:14:06.27462002Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:14:06.274640495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:14:06.27665632Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:14:07.35372525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:14:07.357588792Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:14:07.929113932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:14:07.931604957Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:14:08.162207662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:14:08.164642703Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T04:14:08.617340366Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:14:08.617621873Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:14:08.634822428Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:14:08.634838314Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:14:08.634845975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:14:08.634867765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:14:08.635894433Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-19T04:14:09.42174657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:14:09.424121615Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-19T04:14:09.486288904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:14:10.28822343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:14:10.292252162Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-19T04:14:10.292272609Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:14:10.292291866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:14:10.294182011Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:14:12.067017334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:14:12.073398979Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T04:14:12.386198154Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-19T04:14:13.635287768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:14:14.070469403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37450"
level=debug ts=2026-05-19T04:14:15.829087322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T04:14:16.136115561Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=info ts=2026-05-19T04:14:16.458804866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:14:16.460946703Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:14:16.460965222Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:14:16.460983806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:14:16.462822807Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:14:16.549036393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:14:16.551212639Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:14:16.914505047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:14:16.916798507Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:14:17.441205902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:14:17.443912852Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-19T04:14:18.635846768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:14:19.8862659Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=debug ts=2026-05-19T04:14:21.28017218Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T04:14:23.635926316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:14:23.635950358Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-19T04:14:27.040414106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:14:27.045436614Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T04:14:27.38632204Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-19T04:14:27.555727142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:14:27.558581782Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-19T04:14:28.635455262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:14:29.216152921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:14:29.218822106Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:14:29.218839536Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:14:29.218857577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:14:29.220814506Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:14:29.712802123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:14:29.715414643Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T04:14:31.13630609Z caller=checkpoint.go:372 msg="writing series" size="9.8 kB"
level=info ts=2026-05-19T04:14:32.079171324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:14:32.081976954Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:14:32.081997778Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:14:32.082017168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:14:32.084046294Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:14:33.173921221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:14:33.176236507Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T04:14:33.63518863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:14:34.784245031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:14:34.786732491Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:14:34.786752398Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:14:34.78677428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:14:34.78932537Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:14:34.886109388Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=debug ts=2026-05-19T04:14:34.886122705Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-19T04:14:35.553806023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:14:35.556209807Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:14:35.556233143Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:14:35.556259033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:14:35.558206738Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:14:38.635084406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:14:38.635108122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:14:38.635115919Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:14:38.635147268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:14:38.641561209Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-19T04:14:38.642243998Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006978586s
ts=2026-05-19T04:14:39.487955908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T04:14:41.420319277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:14:41.42467136Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-19T04:14:41.869004773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:14:41.872044856Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:14:41.872065746Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:14:41.87208843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:14:41.875010433Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:14:43.635419111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:14:44.07211419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59576"
level=info ts=2026-05-19T04:14:47.121520723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:14:47.125775068Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-19T04:14:48.001228551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:14:48.004169534Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-19T04:14:48.635748105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:14:48.7791619Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:14:49.443709865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:14:49.447132642Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T04:14:52.93745448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:14:52.943820557Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:14:53.413245184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:14:53.415851446Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=warn ts=2026-05-19T04:14:53.63587428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:14:55.17156594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:14:55.174170186Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:14:55.681685051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:14:55.684330508Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:14:55.684347955Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:14:55.684365638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:14:55.686525256Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:14:57.249727257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:14:57.252579668Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T04:14:57.399134228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T04:14:58.394757772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:14:58.396855376Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:14:58.60158415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:14:58.606137444Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T04:14:58.635670599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:15:02.021068515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:15:02.024937103Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:15:02.034973199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:15:02.037190939Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:15:02.211517702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:15:02.217334158Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:15:02.34457537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:15:02.348235241Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:15:02.348249798Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:15:02.34826911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:15:02.351587749Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:15:02.55812381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:15:02.56037048Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:15:02.560387066Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:15:02.648845423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:15:02.651106584Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T04:15:03.635253513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:15:04.477268448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:15:04.47981843Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:15:04.479835759Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:15:04.47985329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:15:04.481826842Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T04:15:04.968639427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:15:04.970941901Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:15:08.452525912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:15:08.454942549Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T04:15:08.617189747Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:15:08.61826008Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:15:08.618460459Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-19T04:15:08.618498081Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976848 n=1
level=debug ts=2026-05-19T04:15:08.61856128Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976848 n=1
level=info ts=2026-05-19T04:15:08.633331769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:15:08.634696834Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-19T04:15:08.634816206Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-19T04:15:08.634824071Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-19T04:15:08.634834021Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-19T04:15:08.634840513Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.407µs success=true
level=debug ts=2026-05-19T04:15:08.634848955Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:15:08.634856368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:15:08.634881005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:15:08.635581578Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-19T04:15:08.635590324Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=error ts=2026-05-19T04:15:08.635586985Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-19T04:15:09.488948728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=info ts=2026-05-19T04:15:09.635168114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:15:09.63870082Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:15:09.638719314Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-19T04:15:09.638741554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:15:09.641649962Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T04:15:10.42564968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:15:10.427932317Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:15:11.001247186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:15:11.003566288Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-19T04:15:12.385238944Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-19T04:15:12.838850282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:15:12.841649855Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T04:15:13.635575213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:15:14.072830036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54808"
level=info ts=2026-05-19T04:15:15.369931182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:15:15.372441426Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:15:16.136789298Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-19T04:15:16.136809173Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
level=info ts=2026-05-19T04:15:17.311213667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:15:17.316643747Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:15:18.153898747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:15:18.15642579Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-19T04:15:18.635483116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:15:19.047507132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60966"
level=info ts=2026-05-19T04:15:19.160213518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:15:19.162874552Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:15:19.317235808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:15:19.319558181Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T04:15:19.845042277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:15:19.848206357Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T04:15:19.885276351Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-19T04:15:22.079224116Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-19T04:15:23.635365944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:15:23.635367215Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-19T04:15:23.952571148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:15:23.956147928Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T04:15:27.385624533Z caller=checkpoint.go:372 msg="writing series" size="383 kB"
level=warn ts=2026-05-19T04:15:28.635510903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:15:28.856528789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:15:28.860359918Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T04:15:31.136380557Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-19T04:15:31.136402203Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=warn ts=2026-05-19T04:15:33.635324088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:15:34.885352941Z caller=checkpoint.go:372 msg="writing series" size="647 kB"
ts=2026-05-19T04:15:37.311359203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:46914"
level=info ts=2026-05-19T04:15:38.221447051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:15:38.224604224Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-19T04:15:38.635765091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T04:15:38.635766249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:15:38.635787381Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:15:38.635795657Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:15:38.635843587Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-19T04:15:39.161203887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:15:39.164076731Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-19T04:15:39.270237194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:15:39.272704283Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-19T04:15:39.490417229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:15:40.360211835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:15:40.36260601Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-19T04:15:42.385272306Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-19T04:15:43.061350366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:15:43.06521905Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-19T04:15:43.635371332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:15:44.172101212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="629 B" avg_comp="629 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:15:44.174459298Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-19T04:15:44.577505039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:15:44.583520296Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-19T04:15:46.135854408Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-19T04:15:46.987898034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:15:46.994107082Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-19T04:15:48.635548682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:15:49.274226688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:15:49.279085876Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:15:49.885756646Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-19T04:15:50.035331133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:15:50.038769513Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-19T04:15:52.938430329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-19T04:15:53.635306637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:15:53.635477367Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=debug ts=2026-05-19T04:15:53.635492914Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-19T04:15:55.410305463Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:15:57.384739002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-19T04:15:57.384986776Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=error ts=2026-05-19T04:15:57.387681735Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:15:57.944194923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:15:57.946774163Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-19T04:15:58.635466861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:15:58.840223052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:15:58.844382337Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T04:15:59.477459162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:15:59.480587416Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-19T04:16:00.162474172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:16:00.165202355Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:16:01.146400919Z caller=checkpoint.go:372 msg="writing series" size="19 MB"
level=debug ts=2026-05-19T04:16:01.146422205Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=2
level=info ts=2026-05-19T04:16:01.606905588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:16:01.609501621Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-19T04:16:01.860334923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:16:01.862851777Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:16:02.714172735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:16:02.716952641Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-19T04:16:03.635052006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:16:04.885195062Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
ts=2026-05-19T04:16:07.312995321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:51100"
level=info ts=2026-05-19T04:16:08.138641378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:16:08.141736102Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=debug ts=2026-05-19T04:16:08.617266623Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:16:08.618310599Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T04:16:08.635783737Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:16:08.635799235Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:16:08.635807239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:16:08.63580059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:16:08.635912004Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
ts=2026-05-19T04:16:09.492652977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:16:12.38516653Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-19T04:16:12.697947565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:16:12.701310654Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-19T04:16:13.635911949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:16:16.135919293Z caller=checkpoint.go:372 msg="writing series" size="488 kB"
level=info ts=2026-05-19T04:16:16.970052302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:16:16.973417488Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-19T04:16:18.63569523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:16:19.621991299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:16:19.625017837Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T04:16:19.885591357Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-19T04:16:21.297994271Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:16:22.779748443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:16:22.782921679Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-19T04:16:23.635246793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:16:23.635345918Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-19T04:16:27.385641445Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-19T04:16:27.385664977Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-19T04:16:27.628693635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:16:27.630986377Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-19T04:16:28.635346943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:16:30.43390326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:16:30.43781032Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T04:16:31.135638186Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-19T04:16:33.635039968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:16:34.885295002Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-19T04:16:38.34020883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:16:38.346663145Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:16:38.634826053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:16:38.634849439Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:16:38.634857203Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-19T04:16:38.634899758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:16:38.634992671Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-19T04:16:39.231902037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:16:39.23448272Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:16:39.353719046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:16:39.355757097Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-19T04:16:39.495380206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:16:40.124732941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:16:40.130661151Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:16:40.195733903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:16:40.197891792Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:16:41.650199846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:16:41.653684435Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-19T04:16:42.385278579Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-19T04:16:43.404018152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:16:43.406503668Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T04:16:43.635864296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:16:45.490645021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:16:45.49514157Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-19T04:16:46.13531373Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-19T04:16:47.584884329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:16:47.590336203Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-19T04:16:48.635648901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:16:49.885376799Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-19T04:16:50.474300668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:16:50.482202242Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:16:53.235366155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:16:53.237916814Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-19T04:16:53.635761734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:16:53.6358027Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-19T04:16:54.905593379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:16:54.908691791Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-19T04:16:55.016050831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T04:16:55.017551307Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T04:16:55.459718877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:16:55.462139822Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=debug ts=2026-05-19T04:16:55.949603958Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T04:16:57.385820752Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-19T04:16:58.635362505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:16:59.138484231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:16:59.141122506Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-19T04:17:00.9988224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:17:01.00180321Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-19T04:17:01.135436874Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-19T04:17:03.63556499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:17:04.885343598Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-19T04:17:05.202026372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:17:05.204661632Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=info ts=2026-05-19T04:17:07.229102375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:17:07.231708906Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-19T04:17:07.318913942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:17:07.321837334Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T04:17:08.617178847Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-19T04:17:08.618247192Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-19T04:17:08.63528226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-19T04:17:08.635294243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:17:08.635300414Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-19T04:17:08.635310169Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:17:08.635430055Z caller=checkpoint.go:372 msg="writing series" size="275 kB"
ts=2026-05-19T04:17:09.496251656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=info ts=2026-05-19T04:17:11.824966202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:17:11.827748346Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T04:17:12.38505422Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-19T04:17:13.635207617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-19T04:17:14.081167787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34378"
level=info ts=2026-05-19T04:17:15.515955828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:17:15.518562372Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-19T04:17:16.135751345Z caller=checkpoint.go:372 msg="writing series" size="624 kB"
level=debug ts=2026-05-19T04:17:16.135773044Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=13
level=warn ts=2026-05-19T04:17:18.634981333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:17:19.315365116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:17:19.318112201Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T04:17:19.885991007Z caller=checkpoint.go:372 msg="writing series" size="681 kB"
level=warn ts=2026-05-19T04:17:23.635217764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:17:23.635288195Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-19T04:17:25.624890035Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-19T04:17:25.650877318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="694 kB" avg_comp="694 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-19T04:17:25.654507288Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e3e1ae23b%3A19e3e457023%3A1cf1831e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-19T04:17:26.319249477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:17:26.322122307Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:17:26.779296061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="647 B" avg_comp="647 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-19T04:17:26.781952265Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e3e1b7a2e%3A19e3e1b7a2f%3A51626208?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-19T04:17:27.385606281Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-19T04:17:27.565113154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-19T04:17:27.569755482Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e3e1ae31f%3A19e3e3a8e9e%3A41ffbb54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-19T04:17:28.635437842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:17:31.135493335Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-05-19T04:17:32.213566148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.6 MB" avg_uncomp="8.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-19T04:17:32.219968793Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e3e1adfcb%3A19e3e52c4d3%3Ac7b99992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-19T04:17:32.796434741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-19T04:17:32.798808256Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e3e1b7a51%3A19e3e1b7a53%3A5400e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-19T04:17:33.635576894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:17:34.885807453Z caller=checkpoint.go:372 msg="writing series" size="732 kB"
level=debug ts=2026-05-19T04:17:34.88583081Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=info ts=2026-05-19T04:17:35.060790412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="154 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-19T04:17:35.06671037Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e3e1af957%3A19e3e1b6136%3A8337d71?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-19T04:17:36.096467254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.2 MB" total_uncomp="130 MB" avg_uncomp="18 MB" full=5 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-19T04:17:36.10262252Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e3e1ae400%3A19e3e26808a%3A9598ed2b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-19T04:17:36.349243469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-19T04:17:36.352121546Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e3e1add2d%3A19e3e3ac992%3Ab7b70d9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-19T04:17:36.929064016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-19T04:17:36.932183085Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e3e1b7a2e%3A19e3e1b7a53%3A69c05be7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-19T04:17:38.010097747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4260a67c40a0d52d immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"build-e2e-znih\"}"
level=error ts=2026-05-19T04:17:38.012828352Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4260a67c40a0d52d/19e3e25a892%3A19e3e26026b%3A13aca1ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-znih\"}"
level=info ts=2026-05-19T04:17:38.63487306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:17:38.634895449Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T04:17:38.634885216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:17:38.634903301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-19T04:17:38.635975464Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-19T04:17:39.470890056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-19T04:17:39.473334203Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e3e1b64d7%3A19e3e1b64d8%3Ad244644b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-19T04:17:39.497647107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-74723448 10.128.0.79:7946"
level=debug ts=2026-05-19T04:17:42.015492787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T04:17:42.385330863Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-19T04:17:43.635514647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:17:43.64760498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="184 kB" avg_uncomp="184 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-19T04:17:43.650010178Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e3e1ae552%3A19e3e1b7a52%3A7581cb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-19T04:17:44.956300073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-19T04:17:44.958668855Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e3e1b64d6%3A19e3e1bc5ed%3A3577d052?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-19T04:17:46.135810647Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-19T04:17:48.635581432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:17:49.307966706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-19T04:17:49.312201441Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e3e1b64d6%3A19e3e1bc5ed%3A596e165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-19T04:17:49.885817664Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-19T04:17:50.164986108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-19T04:17:50.167673838Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e3e1b7a2e%3A19e3e1b7a2f%3A3c10fd00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-19T04:17:53.63584495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:17:53.636028197Z caller=checkpoint.go:372 msg="writing series" size="451 kB"
level=info ts=2026-05-19T04:17:53.923798351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-19T04:17:53.926508886Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e3e1b64d7%3A19e3e1bc5ed%3A9844d517?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-19T04:17:56.195193433Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-19T04:17:57.385374011Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-19T04:17:58.222935473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4eb3252e47dbc229 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-vesj\"}"
level=error ts=2026-05-19T04:17:58.22557002Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4eb3252e47dbc229/19e3e2571e2%3A19e3e271a26%3Abb541975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-vesj\"}"
level=warn ts=2026-05-19T04:17:58.635560907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-19T04:18:01.135649479Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-19T04:18:03.635554778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:18:04.821286773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-19T04:18:04.824061142Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e3e1adcb0%3A19e3e1bc5ed%3A8ab56a1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-19T04:18:04.885599045Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-19T04:18:08.1580706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-19T04:18:08.160844535Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e3e254565%3A19e3e26899d%3Ae83cf68b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-19T04:18:08.617382455Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-19T04:18:08.617802049Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-19T04:18:08.635826553Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-19T04:18:08.635828856Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-19T04:18:08.635846168Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-19T04:18:08.635849603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-19T04:18:08.635853947Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-19T04:18:09.499801972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=info ts=2026-05-19T04:18:09.817580254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-19T04:18:09.820011646Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e3e1b7a20%3A19e3e1b7a52%3A57ee0451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-19T04:18:11.737506217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-19T04:18:11.740124455Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e3e1bc5ec%3A19e3e1bc5ed%3Ae05da118?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-19T04:18:12.386004041Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-05-19T04:18:13.635296058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"