level=info ts=2026-05-04T22:49:45.265440623Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-04T22:49:45.265484467Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-04T22:49:45.266417627Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-04T22:49:45.267258484Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-04T22:49:45.267687659Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T22:49:45.267763195Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.66 inf=eth0
level=info ts=2026-05-04T22:49:45.268384379Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-bbf91f3f
ts=2026-05-04T22:49:45.268443989Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-04T22:49:45.268456185Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.66 advertisePort=7946
level=debug ts=2026-05-04T22:49:45.26847517Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.66 advertisePort=7946
level=info ts=2026-05-04T22:49:45.268577959Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-04T22:49:45.271253529Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T22:49:45.27129321Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-04T22:49:45.272322535Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-04T22:49:45.272394335Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-05-04T22:49:45.272852733Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T22:49:45.272857269Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=info ts=2026-05-04T22:49:45.272868443Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:49:45.272875965Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-04T22:49:45.27287445Z
level=debug ts=2026-05-04T22:49:45.272897Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=warn ts=2026-05-04T22:49:45.272884599Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.302881ms
level=error ts=2026-05-04T22:49:45.272905792Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-04T22:49:45.272917209Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-05-04T22:49:45.272918176Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-04T22:49:45.272930493Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-04T22:49:45.272935394Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-04T22:49:45.272934429Z
level=debug ts=2026-05-04T22:49:45.272939685Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-05-04T22:49:45.274595922Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T22:49:45.274666754Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.66 inf=eth0
ts=2026-05-04T22:49:45.275340388Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:45.275353755Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:49:45.289531606Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-05-04T22:49:45.289532439Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-04T22:49:45.289549366Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-04T22:49:45.289567566Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-04T22:49:45.289570399Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-04T22:49:45.289587653Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-05-04T22:49:45.289563057Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-05-04T22:49:45.28959353Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-04T22:49:45.289593842Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-04T22:49:45.289605513Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-04T22:49:45.289612155Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-04T22:49:45.28961733Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-04T22:49:45.289820146Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-04T22:49:45.289825199Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-04T22:49:45.289855837Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-04T22:49:45.289877853Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-04T22:49:45.289881715Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-04T22:49:45.289884795Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-04T22:49:45.289914253Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-05-04T22:49:45.289935001Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-04T22:49:45.289950133Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-05-04T22:49:45.289985195Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-04T22:49:45.289998926Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-04T22:49:45.290004743Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-05-04T22:49:45.290010507Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-04T22:49:45.290023661Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-04T22:49:45.290036193Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T22:49:45.290102074Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=25.982µs success=true
level=info ts=2026-05-04T22:49:45.290123368Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-04T22:49:45.29019885Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=178.996µs errors=false
level=info ts=2026-05-04T22:49:45.290206726Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-04T22:49:45.290332909Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=311.322µs errors=false
level=info ts=2026-05-04T22:49:45.290344947Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-04T22:49:45.290350587Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=330.629µs
level=info ts=2026-05-04T22:49:45.290374422Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-04T22:49:45.290426443Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-04T22:49:45.29044629Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-04T22:49:45.290434079Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=23.484281812s
level=info ts=2026-05-04T22:49:45.290528246Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=debug ts=2026-05-04T22:49:45.290541214Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-04T22:49:45.290549862Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-05-04T22:49:45.290543512Z caller=loki.go:615 msg="Loki started" startup_time=88.147218ms
level=debug ts=2026-05-04T22:49:45.290636826Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-04T22:49:46.783531437Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:46.783555193Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T22:49:50.290850189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:49:50.770794743Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:50.770817699Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T22:49:55.290761462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:49:55.492813067Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:55.492850345Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T22:50:00.291228107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:50:05.291166773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:50:08.557644906Z caller=logging.go:137 msg="GET /ready (503) 88.538µs"
ts=2026-05-04T22:50:09.238581196Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.63:7946: dial tcp 10.129.0.63:7946: connect: connection refused"
ts=2026-05-04T22:50:09.239390936Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.62:7946: dial tcp 10.129.0.62:7946: connect: connection refused"
ts=2026-05-04T22:50:09.240520334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.85:7946"
level=info ts=2026-05-04T22:50:09.241475298Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=23.968552298s
level=warn ts=2026-05-04T22:50:10.290903641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:50:15.290798631Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-04T22:50:15.290837281Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-04T22:50:15.290846491Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:50:15.290848283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:50:18.556446534Z caller=logging.go:137 msg="GET /ready (503) 73.277µs"
level=warn ts=2026-05-04T22:50:20.291129626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:50:25.266688526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T22:50:25.29140377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:50:30.291095926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:50:35.291206978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:50:40.290899955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:50:42.401334309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T22:50:42.401387659Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-04T22:50:42.407538553Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T22:50:43.527336881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-04T22:50:45.272518315Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T22:50:45.27272358Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T22:50:45.291085625Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:50:45.291111113Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T22:50:45.291101393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:50:45.291119374Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:50:50.291240025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:50:55.267450742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=warn ts=2026-05-04T22:50:55.290818508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:50:56.185963902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:50:56.197117012Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T22:50:58.244567436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T22:51:00.29121311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:51:05.290853053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:51:07.280071071Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T22:51:10.290880259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:51:14.397965355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-04T22:51:15.291164757Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:51:15.291190843Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:51:15.291198088Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:51:15.29120431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:51:20.29097722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:51:22.143249809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:51:22.151812514Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T22:51:25.268136027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T22:51:25.29058613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:51:30.291620272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:51:35.290952897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:51:36.140782181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:51:39.686425398Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T22:51:40.29101133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:51:45.13754548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40798"
level=info ts=2026-05-04T22:51:45.272512105Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T22:51:45.272518388Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T22:51:45.290824068Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:51:45.290842038Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:51:45.290853989Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:51:45.290862346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:51:49.194243668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49004"
level=warn ts=2026-05-04T22:51:50.290960567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:51:55.269584616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T22:51:55.291476679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:52:00.290932732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:52:04.114304987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T22:52:04.115996248Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:52:05.291186733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:52:07.005698252Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T22:52:10.291172179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:52:15.13838177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:39608"
level=info ts=2026-05-04T22:52:15.290910376Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:52:15.290934255Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T22:52:15.290925977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:52:15.290941709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T22:52:17.142764373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:52:17.151586307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T22:52:20.291002103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:52:24.673215021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:32884"
ts=2026-05-04T22:52:25.271209352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T22:52:25.290962672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:52:30.291160594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:52:35.290701627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:52:36.320936886Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T22:52:40.291293163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:52:45.272624591Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T22:52:45.273021213Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-05-04T22:52:45.291389932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:52:45.291394481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:52:45.291415032Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:52:45.291423362Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T22:52:49.209029391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T22:52:50.291345393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:52:54.674798143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:57098"
ts=2026-05-04T22:52:55.272112886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=warn ts=2026-05-04T22:52:55.29088038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:53:00.29104445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:53:05.290597278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:53:06.142381954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:53:06.150290383Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T22:53:09.984965137Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T22:53:10.291670646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:53:15.291276383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:53:15.291283285Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:53:15.291305157Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:53:15.29131249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:53:20.291249318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:53:24.675742442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55746"
ts=2026-05-04T22:53:25.273552183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T22:53:25.291348944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:53:30.290959186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:53:35.290670335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:53:36.188069741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T22:53:36.188124557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="247 kB" avg_comp="247 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:53:36.194531476Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T22:53:40.290896273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:53:40.750330268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:53:45.272419565Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T22:53:45.27347086Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T22:53:45.290689313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:53:45.290713743Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:53:45.290720997Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:53:45.290729604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:53:50.291480853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:53:52.48972106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:53:52.497052353Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T22:53:54.57219386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="248 kB" avg_comp="248 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:53:54.575995454Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-04T22:53:54.677417544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:57632"
ts=2026-05-04T22:53:55.275200807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T22:53:55.290935507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:54:00.291106039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:54:05.291621119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:54:10.291175832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:54:11.787419948Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T22:54:15.290536105Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:54:15.290561899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:54:15.290570714Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:54:15.290566385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:54:19.205866933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53844"
level=warn ts=2026-05-04T22:54:20.291162165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:54:20.915206497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="248 kB" avg_comp="248 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:54:20.921650193Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-04T22:54:25.27671585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=warn ts=2026-05-04T22:54:25.290656007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:54:30.291052949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:54:35.290620417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:54:37.480172397Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T22:54:39.943837478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:54:39.950885003Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T22:54:40.291445632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:54:45.146682685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40998"
level=info ts=2026-05-04T22:54:45.272593877Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T22:54:45.273206093Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T22:54:45.290494505Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T22:54:45.290545153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:54:45.290566529Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:54:45.29057478Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:54:45.290599933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:54:45.290758284Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-04T22:54:45.290767356Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T22:54:45.290784839Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.277µs success=true
level=debug ts=2026-05-04T22:54:45.2909507Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-04T22:54:49.208159272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56586"
level=debug ts=2026-05-04T22:54:49.646701592Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-04T22:54:50.291477914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:54:50.781035483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:54:54.001058594Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
ts=2026-05-04T22:54:55.278445164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=warn ts=2026-05-04T22:54:55.291154327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:54:58.356436857Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-04T22:55:00.291501329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:55:01.250935443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="248 kB" avg_comp="248 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:55:01.25608792Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T22:55:02.711154927Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=warn ts=2026-05-04T22:55:05.291327975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:07.065734827Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-04T22:55:10.291148772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:11.420131148Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-04T22:55:11.702064463Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T22:55:15.290868437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:55:15.290902962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:55:15.290911743Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:55:15.29093643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:15.778073473Z caller=checkpoint.go:372 msg="writing series" size="4.5 MB"
level=debug ts=2026-05-04T22:55:15.778098983Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.9 MB" series=8
level=debug ts=2026-05-04T22:55:20.130084132Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=warn ts=2026-05-04T22:55:20.291277093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:55:22.637031421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:55:22.645589442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T22:55:24.484880506Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
ts=2026-05-04T22:55:25.279810084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=warn ts=2026-05-04T22:55:25.291484735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:28.839747247Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-04T22:55:30.291659092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:33.195070831Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-04T22:55:35.291599102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:37.550148036Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=warn ts=2026-05-04T22:55:40.290577807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:41.904705161Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-04T22:55:41.955884995Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T22:55:45.272682938Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T22:55:45.273045973Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T22:55:45.291363793Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:55:45.291381598Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:55:45.291391092Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:55:45.291394368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:46.259495039Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=info ts=2026-05-04T22:55:46.977002644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="248 kB" avg_comp="248 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:55:46.980363322Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T22:55:50.291555221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:50.614426179Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=debug ts=2026-05-04T22:55:53.218067587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:55:54.969462467Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
ts=2026-05-04T22:55:55.281220735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=warn ts=2026-05-04T22:55:55.290840145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:55:59.323643331Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T22:56:00.291118119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:03.678200392Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-04T22:56:03.953783813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:56:03.961788442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T22:56:05.291190359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:08.033283817Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-04T22:56:10.291111816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:12.388365182Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-04T22:56:15.2915103Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T22:56:15.291523881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:15.291544464Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:56:15.291555486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T22:56:16.743244884Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-04T22:56:16.9198183Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T22:56:19.214308278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59680"
level=warn ts=2026-05-04T22:56:20.291156067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:21.097601874Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-05-04T22:56:24.687026008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53258"
ts=2026-05-04T22:56:25.283333416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=warn ts=2026-05-04T22:56:25.291468901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:25.452352547Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-04T22:56:27.510785495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="248 kB" avg_comp="248 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:56:27.514233362Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T22:56:29.807419094Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-05-04T22:56:29.807442437Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=17
level=warn ts=2026-05-04T22:56:30.291489263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:34.162486232Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=warn ts=2026-05-04T22:56:35.290885316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:38.517409171Z caller=checkpoint.go:372 msg="writing series" size="7.5 kB"
level=warn ts=2026-05-04T22:56:40.291225266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:42.871738991Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=debug ts=2026-05-04T22:56:44.775829637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T22:56:44.7774815Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T22:56:45.153499964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45270"
level=info ts=2026-05-04T22:56:45.272662658Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T22:56:45.273014928Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T22:56:45.291399947Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T22:56:45.291401761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:45.291418224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:56:45.291426025Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T22:56:47.227315688Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-04T22:56:47.227337516Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=debug ts=2026-05-04T22:56:47.328125359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T22:56:49.215909781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34490"
level=warn ts=2026-05-04T22:56:50.290992679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:51.582123948Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
ts=2026-05-04T22:56:55.284580746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=warn ts=2026-05-04T22:56:55.291386772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:56:55.936506744Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-04T22:56:58.959831449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:56:58.968433542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:56:58.968457874Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T22:56:58.968481161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:56:58.975303186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T22:57:00.291255914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:57:00.291305703Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-04T22:57:04.646113735Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-04T22:57:05.291062552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:57:09.000897497Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-04T22:57:09.951026162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="248 kB" avg_comp="248 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:57:09.957420598Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T22:57:10.291243266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:57:12.037641408Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T22:57:12.580395196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:57:12.588250594Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T22:57:13.356412812Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-04T22:57:15.290613094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:57:15.290646745Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:57:15.290667067Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:57:15.290674647Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T22:57:17.711249557Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
ts=2026-05-04T22:57:19.217626527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44644"
level=warn ts=2026-05-04T22:57:20.291237637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:57:22.065942455Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T22:57:24.691156489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49308"
ts=2026-05-04T22:57:25.285523118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T22:57:25.29057441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:57:26.420100801Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-04T22:57:30.291040734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:57:30.775817587Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-04T22:57:35.130588695Z caller=checkpoint.go:372 msg="writing series" size="344 kB"
level=warn ts=2026-05-04T22:57:35.291149295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:57:37.35889587Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T22:57:38.460980357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T22:57:39.485149586Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-04T22:57:40.291333181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:57:40.331497212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:57:40.340171117Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T22:57:43.839709547Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-04T22:57:45.272561043Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T22:57:45.27256336Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-05-04T22:57:45.290886782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:57:45.290893262Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:57:45.290906208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:57:45.290913755Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T22:57:48.195001441Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-05-04T22:57:50.290875304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:57:52.549442429Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-04T22:57:54.692706091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34680"
ts=2026-05-04T22:57:55.286944029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=warn ts=2026-05-04T22:57:55.290595153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:57:56.90510099Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=debug ts=2026-05-04T22:57:56.905123683Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=16
level=warn ts=2026-05-04T22:58:00.290985823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:01.259353659Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=info ts=2026-05-04T22:58:03.713713567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="248 kB" avg_comp="248 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:58:03.718651572Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T22:58:04.679720054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T22:58:05.291582905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:05.614603596Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T22:58:09.969434663Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-04T22:58:10.291294816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:14.323880227Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=warn ts=2026-05-04T22:58:15.291009378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:58:15.291015669Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:58:15.291039935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:58:15.291047914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T22:58:18.67854248Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-04T22:58:20.291462898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:23.032931093Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-04T22:58:24.69409677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55732"
ts=2026-05-04T22:58:25.288484137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=warn ts=2026-05-04T22:58:25.291404957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:27.388677203Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-04T22:58:30.291427954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:31.743379012Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-04T22:58:32.359363435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T22:58:35.290667973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T22:58:35.926241696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:58:35.934014622Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T22:58:36.097640856Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-04T22:58:36.097681729Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-04T22:58:37.714678475Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T22:58:40.291084152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:40.452883115Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=debug ts=2026-05-04T22:58:44.807697096Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-04T22:58:45.272555306Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T22:58:45.272810128Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T22:58:45.291240312Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:58:45.291279985Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:58:45.291289531Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T22:58:45.291310435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:49.162215132Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-04T22:58:50.291397216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:53.517761807Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
ts=2026-05-04T22:58:55.290649476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=warn ts=2026-05-04T22:58:55.290853511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:58:57.872652188Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-04T22:59:00.290601899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:59:01.922747815Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T22:59:02.226527205Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-04T22:59:03.573799023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="248 kB" avg_comp="248 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:59:03.580313745Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T22:59:05.290959815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:59:06.581751235Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=warn ts=2026-05-04T22:59:10.291027359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:59:10.936457081Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-04T22:59:15.291457786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:59:15.291454266Z caller=checkpoint.go:384 msg="flushing series" totalSize="735 kB" series=8
level=info ts=2026-05-04T22:59:15.291499248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:59:15.291509984Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:59:15.291516928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T22:59:15.297092733Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-04T22:59:15.30069536Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009789251s
level=info ts=2026-05-04T22:59:18.611452039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:59:18.623014932Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T22:59:20.290872911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:59:23.583283578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T22:59:25.291425423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:59:25.292210931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=warn ts=2026-05-04T22:59:30.290964964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T22:59:35.291547854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:59:36.922098431Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T22:59:40.290869387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:59:45.163084382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44256"
level=info ts=2026-05-04T22:59:45.272557241Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T22:59:45.274092994Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T22:59:45.290448397Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T22:59:45.290574355Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-04T22:59:45.290585077Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T22:59:45.290586277Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T22:59:45.29060623Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T22:59:45.290614417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T22:59:45.290617377Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=25.617µs success=true
level=warn ts=2026-05-04T22:59:45.290628237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:59:45.290872718Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T22:59:49.041880374Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-04T22:59:50.29097019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T22:59:52.791471773Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-04T22:59:55.291340775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T22:59:55.295132145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T22:59:56.541876355Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-04T22:59:58.783987967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.0 MB" avg_comp="2.0 MB" total_uncomp="49 MB" avg_uncomp="16 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:59:58.790334419Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:59:58.790359308Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T22:59:58.79039046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.0 MB" avg_comp="2.0 MB" total_uncomp="49 MB" avg_uncomp="16 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T22:59:58.795454646Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T22:59:59.774415453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T22:59:59.782328135Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:00:00.291478027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:00:00.291582599Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-05-04T23:00:00.974873023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:00:04.041063182Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-04T23:00:05.290680259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:00:07.7919525Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-04T23:00:10.291559679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:00:11.540992779Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-04T23:00:15.033676626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.0 MB" avg_comp="2.0 MB" total_uncomp="49 MB" avg_uncomp="16 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:00:15.040519468Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:00:15.290672177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:00:15.290688284Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:00:15.29071069Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:00:15.290718471Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:00:15.295894776Z caller=checkpoint.go:372 msg="writing series" size="5.0 MB"
level=debug ts=2026-05-04T23:00:15.295913348Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.5 MB" series=9
level=debug ts=2026-05-04T23:00:19.041137935Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-04T23:00:20.290701783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:00:20.74229937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:00:22.791634881Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
ts=2026-05-04T23:00:24.701611404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37782"
level=warn ts=2026-05-04T23:00:25.291438754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:00:25.296705435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:00:26.540911382Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-04T23:00:27.9625462Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:00:30.291028295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:00:30.291084672Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-04T23:00:34.041918883Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-04T23:00:35.290908025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:00:37.791394458Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-04T23:00:40.291734755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:00:41.541869297Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
ts=2026-05-04T23:00:45.16660864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60634"
level=info ts=2026-05-04T23:00:45.27316278Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:00:45.273394556Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-04T23:00:45.273458724Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975483 n=1
level=debug ts=2026-05-04T23:00:45.273517Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975483 n=1
level=debug ts=2026-05-04T23:00:45.274329896Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:00:45.290683995Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:00:45.290715418Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:00:45.290726233Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:00:45.290762386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:00:45.291868591Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-04T23:00:49.047495557Z caller=checkpoint.go:372 msg="writing series" size="9.6 MB"
level=debug ts=2026-05-04T23:00:49.047518404Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=9
ts=2026-05-04T23:00:49.232834078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:32870"
level=info ts=2026-05-04T23:00:49.420840711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:00:49.425165689Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:00:49.895634115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:00:49.902895169Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:00:50.291533556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:00:52.791880668Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
ts=2026-05-04T23:00:54.70317999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49224"
level=warn ts=2026-05-04T23:00:55.291350825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:00:55.297734755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:00:56.541720589Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=debug ts=2026-05-04T23:00:57.811117538Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:01:00.291331295Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T23:01:00.291351742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:04.042075662Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=warn ts=2026-05-04T23:01:05.290745625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:07.79159744Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-04T23:01:10.290864445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:11.541496582Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-04T23:01:15.291485197Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T23:01:15.291492089Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:01:15.291526278Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:01:15.291534481Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:01:15.29154516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:19.041883707Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-04T23:01:19.235127731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54662"
level=debug ts=2026-05-04T23:01:19.827876109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:01:19.829889813Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T23:01:20.29097673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:22.791704957Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=debug ts=2026-05-04T23:01:24.261986955Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:01:25.291215677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:01:25.299620918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:01:26.541503838Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-04T23:01:29.951076472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:01:29.956752104Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:01:30.290839545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:30.291010118Z caller=checkpoint.go:372 msg="writing series" size="320 kB"
level=debug ts=2026-05-04T23:01:34.041063037Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=debug ts=2026-05-04T23:01:34.041092691Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-05-04T23:01:35.291054099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:37.791024264Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-04T23:01:40.290636265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:41.541714779Z caller=checkpoint.go:372 msg="writing series" size="991 B"
level=info ts=2026-05-04T23:01:44.752804932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:01:44.764359689Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:01:45.272679537Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:01:45.272688568Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:01:45.291225828Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:01:45.291226871Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-04T23:01:45.291244288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:45.291251328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:01:45.291284211Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:01:49.042368982Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-04T23:01:49.042395264Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-05-04T23:01:50.290667167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:01:51.361451506Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:01:52.791537649Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:01:55.290666484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:01:55.301432775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:01:56.541546276Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-04T23:02:00.291424132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:00.291517763Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-04T23:02:04.022712238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:02:04.041972149Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=warn ts=2026-05-04T23:02:05.292570676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:07.791067801Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T23:02:10.29153289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:11.540977719Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
ts=2026-05-04T23:02:15.172246348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55416"
level=info ts=2026-05-04T23:02:15.291393783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:02:15.291425899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:02:15.291435651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:02:15.291427962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:15.29148226Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-04T23:02:19.041924332Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T23:02:19.23930667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42576"
level=warn ts=2026-05-04T23:02:20.290946604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:22.791570353Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
ts=2026-05-04T23:02:24.708968266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54086"
level=debug ts=2026-05-04T23:02:25.148870868Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:02:25.291046826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:02:25.303390285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:02:26.541501049Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-04T23:02:29.161010568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="96 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:02:29.167945755Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:02:30.290631238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:30.291898785Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T23:02:30.291917663Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=debug ts=2026-05-04T23:02:34.041323887Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-04T23:02:35.29143648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:37.791164626Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-04T23:02:38.747942014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:02:38.75555154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:02:40.291646179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:41.541095147Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-04T23:02:42.550139844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:02:45.272588151Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:02:45.273060095Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:02:45.290563975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:02:45.290606587Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:02:45.290617657Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:02:45.290683392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:45.291832186Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=debug ts=2026-05-04T23:02:49.041919214Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-04T23:02:49.241565479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48792"
level=warn ts=2026-05-04T23:02:50.290610496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:02:52.791419317Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=warn ts=2026-05-04T23:02:55.291278385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:02:55.304627716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:02:56.541536642Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=debug ts=2026-05-04T23:02:56.54156122Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-04T23:02:59.237964582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:03:00.290805124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:03:00.29095567Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T23:03:04.041822289Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=warn ts=2026-05-04T23:03:05.291494353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:03:07.791772537Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T23:03:10.291337202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:03:11.54193551Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
ts=2026-05-04T23:03:15.176297667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53756"
level=info ts=2026-05-04T23:03:15.290964586Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:03:15.291002068Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:03:15.291011405Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:03:15.291072627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:03:15.291076424Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-04T23:03:17.413786137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:03:19.040952917Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-04T23:03:19.243224406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38664"
level=warn ts=2026-05-04T23:03:20.290717862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:03:22.030126954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="96 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:03:22.035302209Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:03:22.791776723Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=warn ts=2026-05-04T23:03:25.290772814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:03:25.306665478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:03:26.541482833Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-04T23:03:26.541507151Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-04T23:03:30.291356273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:03:30.291491171Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-04T23:03:32.719238448Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:03:34.041383542Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-05-04T23:03:35.29134133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:03:36.069032834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:03:36.078125441Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:03:36.078148411Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:03:36.078169726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:03:36.084877853Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:03:37.791013003Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-04T23:03:40.290817576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:03:41.541301952Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-04T23:03:45.272640907Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:03:45.272647236Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:03:45.291178076Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:03:45.291203723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-04T23:03:45.29120379Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-04T23:03:45.291214134Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:03:45.291223065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:03:49.041356125Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
ts=2026-05-04T23:03:49.245444186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53858"
level=warn ts=2026-05-04T23:03:50.292055921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:03:52.791408026Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-04T23:03:54.077233908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:03:54.088487356Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:03:55.29168824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:03:55.308130701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:03:56.542018531Z caller=checkpoint.go:372 msg="writing series" size="355 kB"
level=debug ts=2026-05-04T23:03:58.999724075Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:04:00.291253268Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T23:04:00.291309011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:04:04.04186555Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-04T23:04:05.290707799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:04:07.419320929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:04:07.79166532Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-04T23:04:07.791693694Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=11
level=warn ts=2026-05-04T23:04:10.290787925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:04:11.541489083Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-04T23:04:15.291576225Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:04:15.291624077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:04:15.291636916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:04:15.294049195Z caller=checkpoint.go:384 msg="flushing series" totalSize="66 kB" series=1
level=warn ts=2026-05-04T23:04:15.294245837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:04:15.492585561Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-04T23:04:15.49790977Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.207087006s
level=info ts=2026-05-04T23:04:18.857873022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:04:18.866323038Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:04:19.628896521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:04:19.637115896Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:04:20.290928124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:04:23.460222416Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T23:04:24.717419352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51266"
level=warn ts=2026-05-04T23:04:25.29077089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:04:25.309154302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T23:04:30.291478256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:04:35.291467688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:04:40.291381332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:04:45.273068963Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:04:45.273492043Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:04:45.291106039Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-04T23:04:45.291118154Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-04T23:04:45.291142502Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:04:45.291152762Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:04:45.291195361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:04:45.291309899Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-04T23:04:45.291321875Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:04:45.291348282Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.772µs success=true
level=debug ts=2026-05-04T23:04:45.291569478Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T23:04:47.891740618Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:04:48.709493056Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-04T23:04:50.291193883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:04:52.127946239Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-04T23:04:55.29114108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:04:55.311039585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:04:55.544941501Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-04T23:04:58.963406959Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=warn ts=2026-05-04T23:05:00.291522688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:05:02.380647881Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-04T23:05:04.403525432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:05:04.405072197Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T23:05:05.290692298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:05:05.798582933Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=debug ts=2026-05-04T23:05:09.216067253Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-04T23:05:10.29113349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:05:12.182412836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:05:12.186615266Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:05:12.637652961Z caller=checkpoint.go:372 msg="writing series" size="5.9 MB"
level=debug ts=2026-05-04T23:05:12.637689159Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.5 MB" series=9
level=info ts=2026-05-04T23:05:12.903788939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:05:12.913959116Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:05:15.29053996Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:05:15.29057803Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:05:15.290586195Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:05:15.290642911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:05:16.051765691Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-04T23:05:19.469489582Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=warn ts=2026-05-04T23:05:20.291235272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:05:22.886829459Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-04T23:05:22.901005974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:05:25.291356331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:05:25.312792409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:05:26.305210091Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-04T23:05:29.722023416Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-04T23:05:30.291493684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:05:33.14006256Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-04T23:05:35.290757868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:05:36.558167215Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-04T23:05:39.976010021Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:05:40.290714254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:05:43.412288722Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-04T23:05:43.412325048Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=9
level=info ts=2026-05-04T23:05:45.273509043Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:05:45.274009984Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:05:45.291448078Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:05:45.29149745Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:05:45.291507855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:05:45.291606222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:05:46.810855101Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=debug ts=2026-05-04T23:05:49.134686631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:05:49.254172301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43992"
level=debug ts=2026-05-04T23:05:50.228697763Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-04T23:05:50.290618756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:05:53.475730161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:05:53.491621896Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:05:53.646942856Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-04T23:05:55.293198488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:05:55.314480191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:05:57.064563682Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=debug ts=2026-05-04T23:05:58.305605529Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:06:00.29148836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:06:00.488387216Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-04T23:06:03.90046547Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-05-04T23:06:05.29337038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:06:06.477802476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:06:06.492766729Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:06:07.317455062Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-04T23:06:10.29079443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:06:10.735804064Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-05-04T23:06:14.152882085Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-04T23:06:15.186668995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44594"
level=info ts=2026-05-04T23:06:15.290563644Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:06:15.29060049Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:06:15.290616833Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:06:15.290657571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:06:17.570939447Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-04T23:06:19.255777547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39226"
level=warn ts=2026-05-04T23:06:20.291319784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:06:20.98848215Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-04T23:06:24.406128843Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-05-04T23:06:24.91664676Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:06:25.290596535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:06:25.317414738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:06:27.824241843Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=warn ts=2026-05-04T23:06:30.291228617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:06:31.241822234Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
level=debug ts=2026-05-04T23:06:31.24184291Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-05-04T23:06:31.557959816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:06:34.65980476Z caller=checkpoint.go:372 msg="writing series" size="320 kB"
level=warn ts=2026-05-04T23:06:35.290994073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:06:38.076817717Z caller=checkpoint.go:372 msg="writing series" size="396 kB"
level=warn ts=2026-05-04T23:06:40.290927923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:06:41.494380754Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-04T23:06:44.149502996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:06:44.158693595Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:06:44.912879439Z caller=checkpoint.go:372 msg="writing series" size="991 B"
ts=2026-05-04T23:06:45.189011459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43028"
level=info ts=2026-05-04T23:06:45.272515929Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:06:45.273452051Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:06:45.290972071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:06:45.291003353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:06:45.291015685Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:06:45.291035667Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:06:48.329977215Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-04T23:06:48.603288453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:06:48.607505685Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:06:48.607533491Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:06:48.607565592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:06:48.611108096Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-04T23:06:49.25756942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40564"
level=warn ts=2026-05-04T23:06:50.291511133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:06:50.535873365Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:06:51.74916568Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-04T23:06:51.749192636Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=6
ts=2026-05-04T23:06:54.728287977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:56862"
level=debug ts=2026-05-04T23:06:55.165425001Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:06:55.29071887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:06:55.319111806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:06:58.583572769Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-04T23:07:00.291604213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:07:02.001050638Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-05-04T23:07:05.291619641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:07:05.419108851Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=info ts=2026-05-04T23:07:06.066688051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:07:06.071119114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:07:08.836046842Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-04T23:07:10.291319385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:07:12.254081331Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
ts=2026-05-04T23:07:15.190767236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37646"
level=info ts=2026-05-04T23:07:15.291395283Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:07:15.291427244Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:07:15.291438274Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:07:15.2914536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:07:15.67222795Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-04T23:07:18.210878929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:07:19.08978948Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-04T23:07:19.089813805Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-04T23:07:20.290737962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:07:22.507700442Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-04T23:07:25.290886626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:07:25.320341208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:07:25.925361757Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-04T23:07:26.153843473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:07:26.158827591Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:07:28.348690268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:07:28.355979663Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:07:29.342873127Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-04T23:07:30.291866376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:07:32.760756143Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=debug ts=2026-05-04T23:07:33.339866737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:07:35.290743415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:07:36.178578181Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-04T23:07:39.5960606Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-04T23:07:40.291287032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:07:43.014003305Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=info ts=2026-05-04T23:07:45.273416937Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:07:45.274098241Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-05-04T23:07:45.290613278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:07:45.290639606Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:07:45.29066373Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:07:45.290673041Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:07:46.431128376Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-04T23:07:48.790791553Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:07:49.848706418Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=debug ts=2026-05-04T23:07:49.848728087Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-04T23:07:50.291109843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:07:53.266726314Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-04T23:07:54.732570786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34372"
level=warn ts=2026-05-04T23:07:55.291186795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:07:55.321503816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:07:56.684436731Z caller=checkpoint.go:372 msg="writing series" size="652 kB"
level=debug ts=2026-05-04T23:08:00.10195429Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-04T23:08:00.291174267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:08:03.520567809Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-04T23:08:05.292472285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:08:06.938097746Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T23:08:06.938129272Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-04T23:08:08.789796607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:08:08.799638336Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:08:10.290833139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:08:10.355227329Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=debug ts=2026-05-04T23:08:13.112160737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:08:13.773102968Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-04T23:08:15.193864033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60870"
level=info ts=2026-05-04T23:08:15.291327022Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:08:15.291354363Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T23:08:15.291349844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:08:15.291372543Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:08:17.190651116Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=warn ts=2026-05-04T23:08:20.29092794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:08:20.608489822Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-04T23:08:24.025810479Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-04T23:08:24.585784009Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:08:24.888360775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="143 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:08:24.893062028Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:08:25.296239838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:08:25.323036674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:08:27.444233029Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=warn ts=2026-05-04T23:08:30.290659842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:08:30.862694874Z caller=checkpoint.go:372 msg="writing series" size="672 kB"
level=debug ts=2026-05-04T23:08:30.862718261Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-05-04T23:08:34.280029569Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-04T23:08:35.291420558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:08:37.697350228Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-04T23:08:40.29162045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:08:41.114808024Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-04T23:08:44.532484355Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
ts=2026-05-04T23:08:45.195036836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35306"
level=info ts=2026-05-04T23:08:45.272542222Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:08:45.273967732Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:08:45.291414482Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:08:45.291442846Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:08:45.291452995Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:08:45.291471997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:08:47.95030804Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-04T23:08:50.290854835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:08:51.367687622Z caller=checkpoint.go:372 msg="writing series" size="412 kB"
level=debug ts=2026-05-04T23:08:51.560837782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:08:51.562617675Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:08:54.785989287Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-04T23:08:55.291200195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:08:55.324568791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:08:55.915630936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:08:55.923984752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:08:57.387532161Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:08:58.203967077Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-04T23:08:58.203988017Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-04T23:09:00.291513414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:09:01.621071627Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-04T23:09:05.039090851Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-04T23:09:05.290947185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:09:08.457623683Z caller=checkpoint.go:372 msg="writing series" size="675 kB"
level=warn ts=2026-05-04T23:09:10.291219701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:09:11.874816861Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-04T23:09:15.290720909Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:09:15.290768939Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:09:15.290780279Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:09:15.29085074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:09:15.292029563Z caller=checkpoint.go:384 msg="flushing series" totalSize="836 kB" series=4
level=info ts=2026-05-04T23:09:15.298660484Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-04T23:09:15.303236699Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011722373s
level=info ts=2026-05-04T23:09:19.668342233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:09:19.674638866Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:09:20.291157028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:09:24.93766004Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:09:25.291509454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:09:25.325919645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T23:09:30.291698385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:09:35.291435953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:09:40.290935329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:09:45.198231378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34964"
level=info ts=2026-05-04T23:09:45.273244518Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:09:45.273981139Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:09:45.290488858Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T23:09:45.290633404Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-04T23:09:45.290642949Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:09:45.290661747Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.367µs success=true
level=debug ts=2026-05-04T23:09:45.290854677Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-04T23:09:45.291148777Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:09:45.291173233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:09:45.291180355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:09:45.291205462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:09:47.22075711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:09:47.230151605Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:09:47.230179868Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:09:47.23020681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:09:47.236966577Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:09:48.708865195Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-04T23:09:49.467094564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:09:50.291650184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:09:51.196802222Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:09:52.126861464Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
ts=2026-05-04T23:09:54.743493371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33440"
level=warn ts=2026-05-04T23:09:55.290740133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:09:55.327176952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:09:55.544207999Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-04T23:09:58.961976358Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=warn ts=2026-05-04T23:10:00.291515731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:10:01.062189005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:10:01.072853018Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:10:02.380416961Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-04T23:10:05.291555703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:10:05.797959541Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-04T23:10:09.216558579Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-04T23:10:09.216582066Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=8
level=warn ts=2026-05-04T23:10:10.291649669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:10:12.639445574Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-04T23:10:12.639467619Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.2 MB" series=1
level=info ts=2026-05-04T23:10:15.291144236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:10:15.291192236Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:10:15.291203039Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:10:15.29129267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:10:16.050629765Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-04T23:10:18.010895129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="190 MB" avg_uncomp="21 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:10:18.016656437Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-04T23:10:19.278306744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53608"
level=debug ts=2026-05-04T23:10:19.469139853Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=warn ts=2026-05-04T23:10:20.29070543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:10:22.008022186Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:10:22.885930347Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
ts=2026-05-04T23:10:24.745225185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58208"
level=warn ts=2026-05-04T23:10:25.290959618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:10:25.329206833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:10:26.303989725Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-04T23:10:27.08962805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:10:29.722091922Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-04T23:10:30.291020115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:10:33.139830779Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-04T23:10:35.291593325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:10:36.556987569Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=debug ts=2026-05-04T23:10:39.975061059Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:10:40.291519922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:10:40.99803873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:10:41.012541725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:10:43.426179781Z caller=checkpoint.go:372 msg="writing series" size="27 MB"
level=debug ts=2026-05-04T23:10:43.42620959Z caller=checkpoint.go:384 msg="flushing series" totalSize="28 MB" series=9
level=info ts=2026-05-04T23:10:45.272801406Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:10:45.273191703Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-05-04T23:10:45.290605383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:10:45.290646983Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:10:45.290672517Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:10:45.290682902Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:10:46.810131702Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-04T23:10:50.171902403Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:10:50.228168376Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-04T23:10:50.292322064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:10:53.645416921Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
ts=2026-05-04T23:10:54.746150082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35896"
level=warn ts=2026-05-04T23:10:55.291231828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:10:55.330835716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:10:57.063597453Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T23:11:00.290900506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:11:00.481579676Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=debug ts=2026-05-04T23:11:03.899636855Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-04T23:11:04.664545053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="26 MB" avg_comp="2.6 MB" total_uncomp="214 MB" avg_uncomp="21 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:11:04.669479417Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:11:05.291498576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:11:07.317201934Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=debug ts=2026-05-04T23:11:07.317223694Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-04T23:11:10.291670862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:11:10.734868002Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-04T23:11:14.152344907Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T23:11:15.291102124Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:11:15.291145828Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:11:15.291165237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:11:15.2911711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:11:17.369247974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:11:17.569924599Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-04T23:11:19.282519663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58246"
level=warn ts=2026-05-04T23:11:20.290635437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:11:20.989080058Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=debug ts=2026-05-04T23:11:22.754827471Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:11:24.405815622Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-04T23:11:25.291468035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:11:25.331965653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:11:27.822911604Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=warn ts=2026-05-04T23:11:30.291362469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:11:31.240410429Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=debug ts=2026-05-04T23:11:34.659691771Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=debug ts=2026-05-04T23:11:34.65972286Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-04T23:11:35.290587109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:11:36.618681221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:11:36.629488203Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:11:38.076381373Z caller=checkpoint.go:372 msg="writing series" size="407 kB"
level=warn ts=2026-05-04T23:11:40.291535156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:11:41.494406182Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=debug ts=2026-05-04T23:11:44.911666253Z caller=checkpoint.go:372 msg="writing series" size="991 B"
ts=2026-05-04T23:11:45.205245388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43694"
level=info ts=2026-05-04T23:11:45.272567054Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:11:45.272594708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:11:45.291110857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:11:45.291139835Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T23:11:45.291130211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:11:45.291150193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:11:48.32929504Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
ts=2026-05-04T23:11:49.285091422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38002"
level=warn ts=2026-05-04T23:11:50.291280526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:11:50.683323205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="29 MB" avg_comp="2.7 MB" total_uncomp="237 MB" avg_uncomp="22 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:11:50.692513273Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:11:51.69875136Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:11:51.751780766Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-04T23:11:51.751809843Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=5
level=debug ts=2026-05-04T23:11:55.164836207Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:11:55.291671681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:11:55.333614648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:11:58.582977743Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-04T23:12:00.291423792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:02.000500462Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-04T23:12:04.608665517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:12:05.290754645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:05.419166829Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-04T23:12:05.419193042Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=debug ts=2026-05-04T23:12:08.836385991Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=warn ts=2026-05-04T23:12:10.291446356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:12.253356045Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-04T23:12:15.20681772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57290"
level=info ts=2026-05-04T23:12:15.290630247Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:12:15.290658486Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:12:15.290667232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:12:15.290672697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:15.671636421Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-04T23:12:18.485029712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:12:18.497506657Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:12:19.089214392Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=debug ts=2026-05-04T23:12:19.10024407Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:12:20.290621072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:22.506909697Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-04T23:12:25.290826379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:12:25.335385026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:12:25.92458032Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-04T23:12:29.341675772Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-04T23:12:30.291114055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:32.759835103Z caller=checkpoint.go:372 msg="writing series" size="324 kB"
level=debug ts=2026-05-04T23:12:32.759858046Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-04T23:12:35.29144831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:36.177541848Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=info ts=2026-05-04T23:12:38.265108694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="32 MB" avg_comp="2.7 MB" total_uncomp="261 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:12:38.272428373Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:12:39.595219804Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-04T23:12:40.291484579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:43.013428306Z caller=checkpoint.go:372 msg="writing series" size="497 kB"
ts=2026-05-04T23:12:45.208770584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55340"
level=info ts=2026-05-04T23:12:45.272587523Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:12:45.272671055Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:12:45.291129656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:12:45.291155233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:12:45.291162964Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:12:45.291211989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:46.430744346Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-04T23:12:49.848359024Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-04T23:12:49.848391383Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-04T23:12:50.291164414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:12:50.526590053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:12:50.526566479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:12:50.536849929Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:12:53.265795964Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-04T23:12:54.753156689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:60934"
level=warn ts=2026-05-04T23:12:55.290601173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:12:55.336964694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:12:56.684193449Z caller=checkpoint.go:372 msg="writing series" size="437 kB"
level=debug ts=2026-05-04T23:12:58.08028039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:12:58.082018111Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:13:00.101748463Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=warn ts=2026-05-04T23:13:00.291319068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:03.519843929Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=info ts=2026-05-04T23:13:05.100149532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:13:05.108995972Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:13:05.291607268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:06.936590063Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T23:13:06.936620279Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-04T23:13:08.536785804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:13:08.545581508Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:13:10.291592677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:10.354985029Z caller=checkpoint.go:372 msg="writing series" size="483 kB"
level=debug ts=2026-05-04T23:13:13.771949187Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-04T23:13:15.017485472Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:13:15.291096382Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:13:15.291129082Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:13:15.291139589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:13:15.291135847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:17.190115886Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
ts=2026-05-04T23:13:19.297902212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:36092"
level=warn ts=2026-05-04T23:13:20.290774169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:20.60748342Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-04T23:13:24.025736454Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-04T23:13:25.291308327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:13:25.338540266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:13:27.443818452Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=info ts=2026-05-04T23:13:30.095425756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="32 MB" avg_comp="2.7 MB" total_uncomp="261 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:13:30.101154845Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:13:30.101174584Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:13:30.10120191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="32 MB" avg_comp="2.7 MB" total_uncomp="261 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:13:30.104417327Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:13:30.291444847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:30.862727751Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T23:13:30.86275401Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=7
level=debug ts=2026-05-04T23:13:34.279232051Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-04T23:13:35.291150268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:37.696874007Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=warn ts=2026-05-04T23:13:40.290998017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:41.114381766Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-04T23:13:42.351256907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:13:42.361013072Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:13:42.439280809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:13:44.532440365Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-04T23:13:44.641769564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="32 MB" avg_comp="2.7 MB" total_uncomp="261 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:13:44.646846807Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:13:45.272529557Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:13:45.272560093Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:13:45.291053465Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:13:45.291078834Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:13:45.291086345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:13:45.291104189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:47.949748735Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-04T23:13:50.290612522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:13:51.367559118Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-04T23:13:51.367604535Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-04T23:13:54.785509153Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-04T23:13:55.290962572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:13:55.340693052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:13:58.20282921Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=debug ts=2026-05-04T23:13:59.628365084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:13:59.685771952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:13:59.696761065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:14:00.290789787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:14:01.620585556Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-04T23:14:05.037872403Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-04T23:14:05.290828597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:14:08.456816819Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-04T23:14:08.456844745Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-05-04T23:14:10.291036124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:14:11.401524816Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:14:11.873989184Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
ts=2026-05-04T23:14:15.213020831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55074"
level=info ts=2026-05-04T23:14:15.290646503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:14:15.290678999Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:14:15.290696116Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:14:15.290715338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:14:15.29174611Z caller=checkpoint.go:384 msg="flushing series" totalSize="187 kB" series=1
level=info ts=2026-05-04T23:14:15.297760605Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-04T23:14:15.299414231Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008606206s
level=warn ts=2026-05-04T23:14:20.290962876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:14:22.005436182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="35 MB" avg_comp="2.7 MB" total_uncomp="284 MB" avg_uncomp="22 MB" full=12 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:14:22.012019862Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:14:25.290645531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:14:25.342170553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=warn ts=2026-05-04T23:14:30.290817191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:14:31.021770966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:14:31.029613398Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:14:35.29095201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:14:36.336447633Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:14:39.843301407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:14:39.850632305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:14:40.290892119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:14:45.272531688Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:14:45.272537491Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T23:14:45.291091686Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T23:14:45.291103953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:14:45.291127548Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:14:45.291136473Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:14:45.291137015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:14:45.291410378Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-04T23:14:45.291425828Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:14:45.291450135Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.502µs success=true
level=debug ts=2026-05-04T23:14:45.291615414Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T23:14:48.625072957Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-04T23:14:50.290969783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:14:51.958904902Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
ts=2026-05-04T23:14:54.760090643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38730"
level=warn ts=2026-05-04T23:14:55.291229734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:14:55.292310951Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-04T23:14:55.343715748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:14:55.752274656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:14:58.625878164Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-04T23:15:00.291567945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:01.958841672Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-04T23:15:03.668908431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="35 MB" avg_comp="2.7 MB" total_uncomp="284 MB" avg_uncomp="22 MB" full=12 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:15:03.672531644Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:15:05.291294524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:05.292406257Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-04T23:15:08.626804718Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-04T23:15:08.62683064Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=8
level=debug ts=2026-05-04T23:15:09.71073788Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:15:09.710794602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:15:09.71858726Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:15:10.291083082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:11.962496685Z caller=checkpoint.go:372 msg="writing series" size="7.1 MB"
level=debug ts=2026-05-04T23:15:11.962519219Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.1 MB" series=1
level=info ts=2026-05-04T23:15:15.291080082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:15:15.291098486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:15.291112035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:15:15.291121889Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:15:15.29221115Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-04T23:15:18.625805008Z caller=checkpoint.go:372 msg="writing series" size="444 kB"
level=warn ts=2026-05-04T23:15:20.291583829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:21.958677827Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
ts=2026-05-04T23:15:24.76154455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34878"
level=warn ts=2026-05-04T23:15:25.29068774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:25.291761068Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
ts=2026-05-04T23:15:25.345647045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:15:27.681923569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:15:27.691101391Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:15:28.62533425Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-04T23:15:30.15064476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:15:30.158187183Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:15:30.291376201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:31.959251452Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-04T23:15:35.290951063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:35.292083934Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-04T23:15:38.625721111Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-04T23:15:39.387305908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:15:39.396468607Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:15:40.291538962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:42.000638131Z caller=checkpoint.go:372 msg="writing series" size="37 MB"
level=debug ts=2026-05-04T23:15:42.000664625Z caller=checkpoint.go:384 msg="flushing series" totalSize="38 MB" series=9
level=debug ts=2026-05-04T23:15:43.299010057Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:15:45.272450881Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:15:45.273516864Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:15:45.274209991Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-04T23:15:45.274253135Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975484 n=1
level=debug ts=2026-05-04T23:15:45.274319917Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975484 n=1
level=info ts=2026-05-04T23:15:45.290568582Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:15:45.290611452Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:15:45.29062203Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:15:45.290653498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:45.292667363Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-04T23:15:48.626091311Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
level=warn ts=2026-05-04T23:15:50.291741213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:51.959178494Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-04T23:15:55.291020616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:15:55.292701351Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-04T23:15:55.347593383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:15:56.555105249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:15:57.416565871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="38 MB" avg_comp="2.7 MB" total_uncomp="308 MB" avg_uncomp="22 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:15:57.420346239Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:15:58.625236552Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=info ts=2026-05-04T23:15:58.892237442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:15:58.899394223Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:16:00.291175928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:01.959409184Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-04T23:16:01.959443838Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=warn ts=2026-05-04T23:16:05.291113404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:05.292300858Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=debug ts=2026-05-04T23:16:08.625138118Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-04T23:16:10.290697146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:16:10.464802561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:16:10.474936212Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:16:11.958557151Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-04T23:16:15.221170752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35284"
level=warn ts=2026-05-04T23:16:15.290854402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:16:15.290880593Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:16:15.290919431Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:16:15.290936469Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:16:15.292039222Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T23:16:18.625776186Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-04T23:16:18.84846732Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:16:20.290868314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:21.959184336Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-04T23:16:25.291831651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:25.292109472Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
ts=2026-05-04T23:16:25.34930183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:16:28.625936355Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-04T23:16:30.291099346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:31.958846552Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=debug ts=2026-05-04T23:16:31.95887017Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-04T23:16:35.291340319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:35.293199482Z caller=checkpoint.go:372 msg="writing series" size="728 kB"
level=info ts=2026-05-04T23:16:37.694762201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:16:37.70591954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:16:37.705944259Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:16:37.705969933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:16:37.716135142Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:16:38.626158197Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=debug ts=2026-05-04T23:16:38.626190307Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-05-04T23:16:40.290975994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:41.958650274Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-04T23:16:45.27265481Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:16:45.273058249Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:16:45.291528882Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:16:45.291532539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:45.291548498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:16:45.291556479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:16:45.291622022Z caller=checkpoint.go:372 msg="writing series" size="991 B"
level=debug ts=2026-05-04T23:16:48.625428122Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
ts=2026-05-04T23:16:49.31286184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42010"
level=warn ts=2026-05-04T23:16:50.291526826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:51.958654805Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-04T23:16:52.200145048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="38 MB" avg_comp="2.7 MB" total_uncomp="308 MB" avg_uncomp="22 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:16:52.205311428Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:16:52.701699184Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:16:54.490496788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:16:54.563661915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:16:54.570188574Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:16:55.291334904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:16:55.294532813Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-04T23:16:55.294548797Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=5
ts=2026-05-04T23:16:55.35122545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:16:57.186051471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:16:57.197713682Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:16:58.625223871Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:17:00.290640898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:17:01.435831358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:17:01.445607339Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:17:01.95914507Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-04T23:17:05.29111436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:05.292341501Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=debug ts=2026-05-04T23:17:08.6269428Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-04T23:17:08.626964901Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=4
level=warn ts=2026-05-04T23:17:10.291681712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:11.959112013Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
ts=2026-05-04T23:17:15.225246026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55826"
level=info ts=2026-05-04T23:17:15.290778454Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:17:15.290811899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:17:15.290825071Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:17:15.290813076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:15.291958974Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=debug ts=2026-05-04T23:17:18.625153181Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-04T23:17:18.736612697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:17:18.74614797Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T23:17:19.315073605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39978"
level=warn ts=2026-05-04T23:17:20.291564076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:21.9587269Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-04T23:17:25.290914127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:25.292002018Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T23:17:25.352326322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:17:27.572461749Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:17:28.625136191Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-04T23:17:30.290805991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:31.959139011Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-04T23:17:35.291255292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:35.292662995Z caller=checkpoint.go:372 msg="writing series" size="455 kB"
level=debug ts=2026-05-04T23:17:35.29267771Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-04T23:17:38.016067748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:17:38.028458478Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:17:38.625622258Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=warn ts=2026-05-04T23:17:40.290651566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:40.857396005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:17:40.859232403Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:17:41.958641249Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T23:17:43.430483595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:17:43.439908659Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-bmfg\"}"
ts=2026-05-04T23:17:45.226933498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56446"
level=info ts=2026-05-04T23:17:45.272588917Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:17:45.272984268Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:17:45.291548747Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:17:45.291576807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:17:45.291590599Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:17:45.291596306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:45.291851565Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-04T23:17:48.625458441Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-04T23:17:49.155949188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="38 MB" avg_comp="2.7 MB" total_uncomp="308 MB" avg_uncomp="22 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:17:49.159957751Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:17:50.290650483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:17:51.959119707Z caller=checkpoint.go:372 msg="writing series" size="379 kB"
level=debug ts=2026-05-04T23:17:51.959148688Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-05-04T23:17:53.865417553Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:17:55.291649005Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-04T23:17:55.291649339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:17:55.354034069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:17:58.625994693Z caller=checkpoint.go:372 msg="writing series" size="469 kB"
level=warn ts=2026-05-04T23:18:00.291643173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:01.959471042Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-04T23:18:02.792348046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:18:02.804106128Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:18:05.290675118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:05.292025649Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=debug ts=2026-05-04T23:18:08.62530589Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T23:18:08.625334131Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-04T23:18:10.291291179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:11.959500136Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=info ts=2026-05-04T23:18:15.290827979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:18:15.290859995Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:18:15.290869317Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:18:15.290939266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:15.292018917Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-04T23:18:18.578908005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:18:18.587533826Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:18:18.625626975Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
ts=2026-05-04T23:18:19.320082929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44638"
level=warn ts=2026-05-04T23:18:20.291459093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:21.9592364Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-04T23:18:23.208413554Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:18:25.290586892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:25.292685052Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-04T23:18:25.355075622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:18:28.62591215Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-04T23:18:30.29079576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:31.95913341Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-04T23:18:31.959160522Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=7
level=debug ts=2026-05-04T23:18:34.317702976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:18:35.29100682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:35.29221649Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-04T23:18:38.625748071Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-04T23:18:39.335075866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="41 MB" avg_comp="2.7 MB" total_uncomp="331 MB" avg_uncomp="22 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:18:39.339904707Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:18:40.291001044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:18:41.733601665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:18:41.740398781Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:18:41.959022442Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-04T23:18:45.272540072Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:18:45.272552684Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:18:45.291060866Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:18:45.291086521Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:18:45.291093483Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:18:45.291084166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:45.292176962Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-04T23:18:47.477431392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:18:47.486382184Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:18:48.625625197Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
ts=2026-05-04T23:18:49.322375012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49920"
level=warn ts=2026-05-04T23:18:50.290993416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:51.959664595Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T23:18:51.959688323Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-05-04T23:18:52.157050857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:18:55.290737226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:18:55.291958067Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
ts=2026-05-04T23:18:55.356068615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:18:58.626214176Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=warn ts=2026-05-04T23:19:00.291257095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:19:01.958549914Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T23:19:05.29069779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:19:05.291819647Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-04T23:19:08.626459907Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T23:19:08.626494945Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-04T23:19:09.934390173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:19:09.943460563Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:19:10.291339998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:19:11.958982221Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=info ts=2026-05-04T23:19:15.290892397Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:19:15.290896259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:19:15.290924049Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:19:15.290932212Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:19:15.292003156Z caller=checkpoint.go:384 msg="flushing series" totalSize="212 kB" series=1
level=info ts=2026-05-04T23:19:15.297983951Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-04T23:19:15.300027329Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008456409s
ts=2026-05-04T23:19:19.323969164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57504"
level=warn ts=2026-05-04T23:19:20.291209963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:19:23.932312624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:19:25.204297325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="41 MB" avg_comp="2.7 MB" total_uncomp="331 MB" avg_uncomp="22 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:19:25.208444589Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:19:25.291122088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:19:25.357636651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:19:27.04373681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:19:29.984021663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:19:29.992492347Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:19:29.992521468Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:19:29.992547878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:19:30.000466947Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:19:30.291217458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:19:35.291197117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:19:38.004017091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:19:38.011561673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:19:40.291343187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:19:45.272520751Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:19:45.272897407Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:19:45.290474873Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T23:19:45.290535571Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:19:45.290564626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:19:45.290574895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:19:45.290592229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:19:45.290622406Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-04T23:19:45.290634276Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:19:45.290656286Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.421µs success=true
level=debug ts=2026-05-04T23:19:45.290854357Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T23:19:48.624582676Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-04T23:19:48.852321701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:19:48.861627784Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:19:50.2910653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:19:50.439783127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:19:50.447618108Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:19:51.958060786Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=debug ts=2026-05-04T23:19:55.291222461Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-04T23:19:55.291280444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:19:55.359652861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:19:58.624723908Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=debug ts=2026-05-04T23:19:59.009995432Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:20:00.290885883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:01.958056158Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-04T23:20:05.291107395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:05.291233034Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-04T23:20:07.172995851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:20:08.632462971Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-04T23:20:08.632487595Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.6 MB" series=8
level=warn ts=2026-05-04T23:20:10.291629665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:20:10.928939545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:20:10.936544703Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:20:11.966406435Z caller=checkpoint.go:372 msg="writing series" size="7.6 MB"
level=debug ts=2026-05-04T23:20:11.966431402Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.6 MB" series=1
level=info ts=2026-05-04T23:20:14.912274961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:20:14.917393405Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:20:14.917417101Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:20:14.917451685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:20:14.920829682Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:20:15.291342038Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-04T23:20:15.291368349Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:20:15.291379828Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:20:15.291386904Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:20:15.291380641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:18.624712567Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
ts=2026-05-04T23:20:19.328203025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50752"
level=warn ts=2026-05-04T23:20:20.291205014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:21.957747696Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-04T23:20:23.440164758Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:20:23.440223229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:20:23.445870137Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:20:25.291321843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:25.291351184Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
ts=2026-05-04T23:20:25.360761816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:20:28.625819809Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T23:20:28.625841686Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-05-04T23:20:30.291464797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:20:30.896773988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:20:30.904156444Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:20:31.957858718Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-04T23:20:32.94422842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:20:32.952158365Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:20:34.337348805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:20:34.341169116Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:20:35.280026763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:20:35.284506636Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:20:35.290569985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:35.291764335Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-04T23:20:38.624224336Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:20:40.290852102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:41.997242454Z caller=checkpoint.go:372 msg="writing series" size="44 MB"
level=debug ts=2026-05-04T23:20:41.997283487Z caller=checkpoint.go:384 msg="flushing series" totalSize="44 MB" series=4
ts=2026-05-04T23:20:45.236779429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38252"
level=info ts=2026-05-04T23:20:45.272610448Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:20:45.272679066Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:20:45.291217373Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:20:45.291238453Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:20:45.291246486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:20:45.291324966Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-04T23:20:45.291355234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:48.625977512Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=warn ts=2026-05-04T23:20:50.291312528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:51.131990242Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:20:51.416852669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:20:51.42720594Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:20:51.95802954Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
ts=2026-05-04T23:20:54.782639969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58758"
level=warn ts=2026-05-04T23:20:55.291478466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:20:55.291700845Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-04T23:20:55.291718055Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-04T23:20:55.361905883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:20:55.664462807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:20:58.624226937Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T23:21:00.29142916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:21:01.924950626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:21:01.929706838Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:21:01.95840166Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=warn ts=2026-05-04T23:21:05.291421965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:05.29156692Z caller=checkpoint.go:372 msg="writing series" size="349 kB"
level=debug ts=2026-05-04T23:21:08.625343353Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-04T23:21:08.625365998Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-04T23:21:08.748550266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:21:08.752888141Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:21:10.291088694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:11.957768941Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=info ts=2026-05-04T23:21:15.291114885Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:21:15.291142004Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:21:15.291149688Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:21:15.291152634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:15.29123401Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T23:21:18.624332977Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-04T23:21:19.730699223Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:21:20.291590863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:21.957946744Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-04T23:21:25.290608299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:25.291823872Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=info ts=2026-05-04T23:21:25.359906826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
ts=2026-05-04T23:21:25.363372251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=error ts=2026-05-04T23:21:25.369664128Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:21:28.624999483Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=info ts=2026-05-04T23:21:29.811298011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:21:29.819072877Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:21:29.819099143Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:21:29.819122287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:21:29.825465213Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:21:30.291117983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:31.957996248Z caller=checkpoint.go:372 msg="writing series" size="438 kB"
level=debug ts=2026-05-04T23:21:32.841909517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:21:32.843769547Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T23:21:35.291542061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:35.292371535Z caller=checkpoint.go:372 msg="writing series" size="937 kB"
level=debug ts=2026-05-04T23:21:35.292389997Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=8
level=debug ts=2026-05-04T23:21:38.625517288Z caller=checkpoint.go:372 msg="writing series" size="701 kB"
level=info ts=2026-05-04T23:21:39.064147657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:21:39.072478334Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:21:40.291608716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:41.958033356Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-04T23:21:45.240572179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48856"
level=info ts=2026-05-04T23:21:45.272526476Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:21:45.272927924Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:21:45.291430331Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:21:45.291457494Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:21:45.291466227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:21:45.291498871Z caller=checkpoint.go:372 msg="writing series" size="991 B"
level=warn ts=2026-05-04T23:21:45.291528947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:21:46.829026823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:21:46.833455757Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:21:48.625103886Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-04T23:21:49.278384057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:21:49.288804845Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:21:50.234250439Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:21:50.291554339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:51.958196225Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-04T23:21:54.311901506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:21:54.317909876Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-04T23:21:54.786226519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46032"
level=warn ts=2026-05-04T23:21:55.291315815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:21:55.293098038Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-04T23:21:55.293113659Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=6
ts=2026-05-04T23:21:55.364800334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:21:58.624422435Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:22:00.29110037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:01.958419617Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-04T23:22:05.290612431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:05.291872924Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=debug ts=2026-05-04T23:22:08.625114687Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-04T23:22:08.625143696Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=4
level=warn ts=2026-05-04T23:22:10.291243898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:11.958852971Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-04T23:22:13.303325317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:22:13.312315472Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:22:15.291257818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:22:15.29125592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:22:15.291297338Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-04T23:22:15.291295961Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-04T23:22:15.291309998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:22:18.377644243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:22:18.624249155Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-04T23:22:19.785206427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:22:19.792398223Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:22:19.941889497Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:22:20.291051698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:21.958631877Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-04T23:22:24.697318307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:22:24.707248286Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
ts=2026-05-04T23:22:24.787714204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46518"
level=debug ts=2026-05-04T23:22:25.291111002Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-04T23:22:25.291135222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:22:25.367094241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:22:28.624287949Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-04T23:22:30.291015375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:31.957811115Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-04T23:22:35.290877848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:35.29104771Z caller=checkpoint.go:372 msg="writing series" size="379 kB"
level=debug ts=2026-05-04T23:22:35.291064501Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-05-04T23:22:38.625045149Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-04T23:22:40.291209183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:41.95819598Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T23:22:45.09088275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:22:45.095858896Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:22:45.272494426Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:22:45.272562669Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T23:22:45.291097973Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:22:45.291117805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:22:45.291125347Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:22:45.291122256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:45.29134208Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-04T23:22:45.499423144Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:22:46.767665499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="46 MB" avg_comp="2.7 MB" total_uncomp="379 MB" avg_uncomp="22 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:22:46.772893871Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:22:48.624459385Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-04T23:22:50.290985554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:51.957963603Z caller=checkpoint.go:372 msg="writing series" size="424 kB"
level=debug ts=2026-05-04T23:22:51.957989304Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
ts=2026-05-04T23:22:54.788793221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40158"
level=warn ts=2026-05-04T23:22:55.291568778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:22:55.291603476Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-04T23:22:55.369285328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:22:58.625331071Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=warn ts=2026-05-04T23:23:00.291256516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:23:01.958185599Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=warn ts=2026-05-04T23:23:05.290862027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:23:05.290941694Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=info ts=2026-05-04T23:23:07.03341483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:23:07.041718822Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:23:08.624764438Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T23:23:08.624796383Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-04T23:23:09.616873027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:23:09.62469303Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:23:09.624717879Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:23:09.624742533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:23:09.631455367Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:23:10.291328512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:23:11.958323707Z caller=checkpoint.go:372 msg="writing series" size="664 kB"
ts=2026-05-04T23:23:15.244952859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53168"
level=info ts=2026-05-04T23:23:15.291470255Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:23:15.29147655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:23:15.291504666Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-04T23:23:15.291501152Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-04T23:23:15.291516911Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T23:23:16.796079745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:23:16.805571535Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:23:16.942918152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:23:18.027730345Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:23:18.624430049Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=warn ts=2026-05-04T23:23:20.2908453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:23:21.95342334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:23:21.95796222Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=error ts=2026-05-04T23:23:21.961707567Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:23:25.291290669Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-04T23:23:25.29131299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:23:25.370778528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:23:28.624435635Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=warn ts=2026-05-04T23:23:30.29129725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:23:31.959771892Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-04T23:23:31.959805222Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=7
level=info ts=2026-05-04T23:23:32.741295151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:23:32.74965414Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:23:35.290968064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:23:35.291049155Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-04T23:23:38.624437157Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-04T23:23:40.291034835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:23:41.958101102Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-04T23:23:42.886734934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="46 MB" avg_comp="2.7 MB" total_uncomp="379 MB" avg_uncomp="22 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:23:42.891220033Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:23:45.272558093Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:23:45.272762132Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-04T23:23:45.291325939Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-04T23:23:45.291324099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:23:45.291323768Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:23:45.291351359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:23:45.29136159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:23:48.624480937Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-04T23:23:50.171030544Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:23:50.17910641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:23:50.187458933Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:23:50.291247063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:23:51.436682015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:23:51.958663605Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T23:23:51.958685836Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-04T23:23:54.792514284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42912"
level=warn ts=2026-05-04T23:23:55.291217871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:23:55.291304022Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
ts=2026-05-04T23:23:55.372368406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:23:58.62494614Z caller=checkpoint.go:372 msg="writing series" size="428 kB"
level=warn ts=2026-05-04T23:24:00.291537907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:24:01.958545181Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-04T23:24:03.187287368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:24:03.198197693Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:24:05.291363518Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-04T23:24:05.291367275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:24:08.625310539Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T23:24:08.625333448Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=warn ts=2026-05-04T23:24:10.291099262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:24:11.95837324Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-04T23:24:13.755581127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:24:13.76612644Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=info ts=2026-05-04T23:24:15.290787814Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:24:15.290803361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:24:15.29081572Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:24:15.290830086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:24:15.290831829Z caller=checkpoint.go:384 msg="flushing series" totalSize="242 kB" series=1
level=info ts=2026-05-04T23:24:15.296297925Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-04T23:24:15.298637596Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007826655s
level=debug ts=2026-05-04T23:24:17.355473826Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:24:20.29134096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:24:25.291298416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:24:25.375340956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:24:26.142342215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:24:29.54429601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="46 MB" avg_comp="2.7 MB" total_uncomp="379 MB" avg_uncomp="22 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:24:29.54777479Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:24:30.290834394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:24:30.881580672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:24:30.885653637Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:24:35.291102544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:24:38.075997376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:24:38.08325839Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:24:40.290688548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:24:45.272519822Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:24:45.272526285Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-04T23:24:45.290961413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:24:45.290958317Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-04T23:24:45.290974889Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-04T23:24:45.290995278Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:24:45.291006092Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T23:24:45.291312962Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-04T23:24:45.291324058Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:24:45.29135456Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=25.562µs success=true
level=debug ts=2026-05-04T23:24:45.291546923Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T23:24:48.625363486Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
ts=2026-05-04T23:24:49.345862907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40316"
level=warn ts=2026-05-04T23:24:50.290940373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:24:51.151200775Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:24:51.958541858Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-04T23:24:54.802830175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:24:54.811241739Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:24:55.290672395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:24:55.291851549Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-04T23:24:55.377351266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:24:58.62582883Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-04T23:25:00.291697443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:01.958894355Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-04T23:25:05.035905584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:25:05.044121831Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:25:05.290693397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:05.291845523Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-04T23:25:08.630349577Z caller=checkpoint.go:372 msg="writing series" size="7.8 MB"
level=debug ts=2026-05-04T23:25:08.630375277Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.3 MB" series=8
level=warn ts=2026-05-04T23:25:10.290753704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:11.963191218Z caller=checkpoint.go:372 msg="writing series" size="8.1 MB"
level=debug ts=2026-05-04T23:25:11.963216676Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.1 MB" series=1
level=info ts=2026-05-04T23:25:15.29054081Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:25:15.29056765Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:25:15.290575032Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:25:15.290631412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:15.291603327Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-04T23:25:17.162317626Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:25:18.625225509Z caller=checkpoint.go:372 msg="writing series" size="355 kB"
ts=2026-05-04T23:25:19.348043412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49560"
level=debug ts=2026-05-04T23:25:19.664802566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:25:19.666683406Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T23:25:20.291281517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:25:20.733929596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:25:20.73901056Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:25:21.958297228Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
ts=2026-05-04T23:25:24.796292995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34490"
level=warn ts=2026-05-04T23:25:25.291400887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:25.292545269Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
ts=2026-05-04T23:25:25.379068798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-04T23:25:26.534156913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="49 MB" avg_comp="2.7 MB" total_uncomp="403 MB" avg_uncomp="22 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:25:26.538542319Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:25:28.626904357Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T23:25:28.6269273Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-05-04T23:25:30.291343184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:31.958554015Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-04T23:25:32.46640485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:25:32.474644592Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:25:35.290634718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:35.291767756Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-04T23:25:38.625199372Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:25:40.29112554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:42.004495696Z caller=checkpoint.go:372 msg="writing series" size="51 MB"
level=debug ts=2026-05-04T23:25:42.004533336Z caller=checkpoint.go:384 msg="flushing series" totalSize="51 MB" series=4
level=debug ts=2026-05-04T23:25:44.00211682Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:25:44.793382171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:25:44.805246803Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:25:45.272539164Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:25:45.272837528Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:25:45.291287618Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:25:45.291309473Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:25:45.291318923Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:25:45.291367884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:45.292487612Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-05-04T23:25:48.625602443Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=info ts=2026-05-04T23:25:50.11640399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:25:50.124500397Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:25:50.1245242Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:25:50.124548791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:25:50.131565655Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:25:50.291047072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:51.958477958Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=warn ts=2026-05-04T23:25:55.290994163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:25:55.292302563Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-04T23:25:55.292317074Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-04T23:25:55.380600583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:25:58.625612934Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T23:26:00.291623438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:01.959463948Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-04T23:26:02.158655427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:26:02.169481437Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:26:05.290672708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:05.292075766Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-04T23:26:08.625201412Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-04T23:26:10.291032169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:11.959194162Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-04T23:26:11.959231432Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-04T23:26:15.291214347Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:26:15.2912421Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:26:15.291249885Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:26:15.29125026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:15.292335884Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T23:26:15.61389664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="49 MB" avg_comp="2.7 MB" total_uncomp="403 MB" avg_uncomp="22 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:26:15.617529986Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-04T23:26:15.621527704Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:26:16.870632403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:26:16.87515216Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:26:18.599942812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:26:18.607529006Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:26:18.625654689Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-04T23:26:20.291010168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:21.647928022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:26:21.959154843Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T23:26:25.291603243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:25.291709847Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
ts=2026-05-04T23:26:25.382180707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:26:28.625127382Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=warn ts=2026-05-04T23:26:30.290579562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:26:31.246587873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:26:31.254849621Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:26:31.959788531Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-04T23:26:31.959812908Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-04T23:26:33.295923284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:26:33.304669423Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:26:35.291503094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:35.292912148Z caller=checkpoint.go:372 msg="writing series" size="870 kB"
level=debug ts=2026-05-04T23:26:38.625875613Z caller=checkpoint.go:372 msg="writing series" size="818 kB"
level=debug ts=2026-05-04T23:26:38.625899084Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=warn ts=2026-05-04T23:26:40.290999526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:41.57582189Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:26:41.958943151Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
ts=2026-05-04T23:26:45.255695002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:50318"
level=info ts=2026-05-04T23:26:45.273540075Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:26:45.274048424Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:26:45.291388146Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:26:45.291389953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:45.291412437Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:26:45.291420539Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:26:45.292592314Z caller=checkpoint.go:372 msg="writing series" size="991 B"
level=debug ts=2026-05-04T23:26:48.625302788Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-04T23:26:50.291298825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:51.958331539Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-04T23:26:55.29103242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:26:55.295329632Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-04T23:26:55.295363057Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=5
ts=2026-05-04T23:26:55.383799569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:26:58.625594784Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:27:00.291382814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:27:01.95937236Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-04T23:27:05.29141753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:27:05.292671693Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-04T23:27:07.407379212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:27:08.628892252Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-04T23:27:08.62892141Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=warn ts=2026-05-04T23:27:10.29129779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:27:11.959243258Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-04T23:27:11.988468654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:27:11.99472386Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:27:11.994752924Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:27:11.994783779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:27:11.999432063Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:27:14.418094636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="49 MB" avg_comp="2.7 MB" total_uncomp="403 MB" avg_uncomp="22 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:27:14.422589514Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:27:14.422616665Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:27:14.422664626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="49 MB" avg_comp="2.7 MB" total_uncomp="403 MB" avg_uncomp="22 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:27:14.426738091Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:27:14.569249417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:27:14.577940896Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T23:27:15.256990629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54216"
level=warn ts=2026-05-04T23:27:15.291353642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:27:15.29438253Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:27:15.294395957Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=debug ts=2026-05-04T23:27:15.294406879Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:27:15.294417786Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:27:17.043497826Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:27:18.625941882Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-04T23:27:20.296333659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:27:20.464494477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:27:20.472059767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:27:21.958601417Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-04T23:27:23.76525468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:27:23.77569161Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-bmfg\"}"
level=info ts=2026-05-04T23:27:24.922942928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:27:24.927240995Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:27:25.292982912Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-04T23:27:25.293056478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:27:25.386664085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:27:28.627357272Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-04T23:27:30.291433681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:27:31.944117395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:27:31.949448502Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:27:31.95854767Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-04T23:27:35.292805674Z caller=checkpoint.go:372 msg="writing series" size="463 kB"
level=debug ts=2026-05-04T23:27:35.292831881Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-04T23:27:35.295340265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:27:38.625321647Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-04T23:27:40.291391201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:27:41.958526358Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-04T23:27:43.519544719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:27:45.272538617Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:27:45.273281444Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:27:45.291420915Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:27:45.291452711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:27:45.29147302Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:27:45.291516221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:27:45.291872375Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-04T23:27:48.626083839Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-04T23:27:50.29335024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:27:51.958996254Z caller=checkpoint.go:372 msg="writing series" size="621 kB"
level=debug ts=2026-05-04T23:27:51.959019774Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-05-04T23:27:52.810472625Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:27:55.293347482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:27:55.295360328Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-04T23:27:55.395466809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:27:58.365191393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:27:58.374854281Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:27:58.625643426Z caller=checkpoint.go:372 msg="writing series" size="678 kB"
level=warn ts=2026-05-04T23:28:00.29237027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:28:01.959327303Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-04T23:28:04.058520377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:28:04.064925925Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:28:05.293379504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:28:05.293688254Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=debug ts=2026-05-04T23:28:05.293705663Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-04T23:28:08.061126257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:28:08.071049137Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:28:08.071079957Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:28:08.071109387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:28:08.078425203Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:28:08.625894793Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-04T23:28:09.885376198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:28:09.919530559Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:28:10.293356182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:28:11.937488371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:28:11.942396532Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:28:11.95887059Z caller=checkpoint.go:372 msg="writing series" size="561 kB"
ts=2026-05-04T23:28:15.259731877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43228"
level=warn ts=2026-05-04T23:28:15.30037405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:28:15.303656004Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:28:15.303701989Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:28:15.303713384Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:28:15.303841729Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-04T23:28:18.625915622Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
ts=2026-05-04T23:28:19.360370807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57816"
level=warn ts=2026-05-04T23:28:20.291404437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:28:21.958650716Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T23:28:24.494480634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:28:24.514156062Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:28:25.291606202Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-04T23:28:25.291671264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:28:25.413545006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:28:26.15303178Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:28:26.153120156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:28:26.161525375Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:28:28.625689653Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-04T23:28:29.021773711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:28:30.293349383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:28:31.960938553Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-04T23:28:31.96096086Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=8
level=warn ts=2026-05-04T23:28:35.293095835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:28:35.293528997Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-04T23:28:38.625795175Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-04T23:28:40.291360119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:28:41.958940241Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-04T23:28:42.909637114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:28:42.919373719Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:28:44.220060212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:28:44.22407913Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:28:45.272354892Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:28:45.273406921Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-04T23:28:45.291446712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:28:45.291686756Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-04T23:28:45.291735484Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:28:45.29175003Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:28:45.29175942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:28:48.625627495Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-04T23:28:50.294347348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:28:50.307124716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:28:50.316384014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:28:51.959620634Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T23:28:51.959643392Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-04T23:28:54.095495297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:28:54.107749401Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:28:55.293140574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:28:55.29751486Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-04T23:28:55.415871051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:28:56.22268836Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:28:57.134743138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:28:57.142211091Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:28:58.625924706Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=warn ts=2026-05-04T23:29:00.297336503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:29:01.959148608Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-04T23:29:03.619848124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:29:03.628874291Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:29:05.292540112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:29:05.292652209Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-04T23:29:08.627225889Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-04T23:29:08.62724915Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=warn ts=2026-05-04T23:29:10.293208373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:29:11.959072705Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-04T23:29:15.292312881Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:29:15.292356436Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:29:15.292370279Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:29:15.292498477Z caller=checkpoint.go:384 msg="flushing series" totalSize="69 kB" series=1
level=warn ts=2026-05-04T23:29:15.292595676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:29:15.30101704Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-04T23:29:15.304055024Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012549562s
level=info ts=2026-05-04T23:29:16.558496925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:29:16.569319292Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T23:29:19.363919126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38326"
level=warn ts=2026-05-04T23:29:20.294340628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:29:24.819526497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34960"
level=warn ts=2026-05-04T23:29:25.29107895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:29:25.41775885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:29:29.414810006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:29:29.416710461Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T23:29:30.291387037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:29:31.288414134Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:29:35.293079252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:29:40.292338501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:29:41.703168186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:29:41.720678502Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:29:41.720714872Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:29:41.720746902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:29:41.729895018Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:29:43.897409082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:29:43.91157922Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-04T23:29:45.264638783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44188"
level=info ts=2026-05-04T23:29:45.27276005Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:29:45.273162171Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:29:45.291437259Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:29:45.291472884Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:29:45.291483678Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T23:29:45.291545135Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T23:29:45.291679521Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-04T23:29:45.291689003Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:29:45.291715655Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.209µs success=true
level=debug ts=2026-05-04T23:29:45.291977648Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-04T23:29:45.292244147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:29:48.625634213Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-04T23:29:50.295360279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:29:51.693375368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:29:51.706437594Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:29:51.958871071Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-04T23:29:55.132938415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:29:55.142023318Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:29:55.292076348Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-04T23:29:55.293872808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:29:55.420078666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-04T23:29:57.126346052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:29:57.134918949Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
level=info ts=2026-05-04T23:29:57.579487305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:29:57.595163399Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:29:58.626436038Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=debug ts=2026-05-04T23:29:59.306578654Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:30:00.290674714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:01.959097419Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-04T23:30:05.290925707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:05.293497707Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-05-04T23:30:08.637647487Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-04T23:30:08.637674195Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=8
level=warn ts=2026-05-04T23:30:10.291315209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:11.965146155Z caller=checkpoint.go:372 msg="writing series" size="8.6 MB"
level=debug ts=2026-05-04T23:30:11.96517897Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.6 MB" series=1
level=info ts=2026-05-04T23:30:14.356193936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:30:14.366167233Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T23:30:15.265890289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45546"
level=info ts=2026-05-04T23:30:15.290719944Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:30:15.290764972Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:30:15.290776919Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:30:15.290811893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:15.293035254Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-04T23:30:18.626773893Z caller=checkpoint.go:372 msg="writing series" size="608 kB"
ts=2026-05-04T23:30:19.3686052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53426"
level=warn ts=2026-05-04T23:30:20.290984098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:21.959535581Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
ts=2026-05-04T23:30:24.823438085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55496"
level=warn ts=2026-05-04T23:30:25.290813747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:25.292996807Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
ts=2026-05-04T23:30:25.421722519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:30:28.627474326Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T23:30:28.627499995Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=debug ts=2026-05-04T23:30:30.046457079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:30:30.291546997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:31.958751711Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-04T23:30:32.446403339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:30:32.455780334Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:30:34.817671729Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:30:35.290826785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:35.29312561Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=debug ts=2026-05-04T23:30:38.625940452Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:30:40.291079436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:30:42.005920617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:30:42.012394481Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:30:42.019131445Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-04T23:30:42.019160945Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=4
level=info ts=2026-05-04T23:30:42.418003376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:30:42.42243435Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-04T23:30:45.266924006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56238"
level=info ts=2026-05-04T23:30:45.272700306Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:30:45.272711099Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-04T23:30:45.274020703Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-04T23:30:45.27407576Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975485 n=1
level=debug ts=2026-05-04T23:30:45.274123527Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975485 n=1
level=info ts=2026-05-04T23:30:45.291176527Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:30:45.291213487Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:30:45.291224324Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:30:45.291234118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:45.292441219Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-04T23:30:46.959146268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:30:46.968112338Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:30:48.626843406Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=info ts=2026-05-04T23:30:49.410864794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:30:49.419416172Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:30:50.291075941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:51.959688233Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=warn ts=2026-05-04T23:30:55.291076534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:30:55.292373962Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-04T23:30:55.292390182Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-04T23:30:55.423720579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:30:58.626007215Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T23:31:00.291116981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:31:00.399476473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:31:00.409185784Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:31:01.960157701Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=warn ts=2026-05-04T23:31:05.291103499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:05.292485374Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=debug ts=2026-05-04T23:31:06.486824693Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:31:08.625528849Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-04T23:31:08.625554278Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-04T23:31:10.29132226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:11.959130716Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-04T23:31:13.112604982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:31:13.122881701Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:31:15.291278236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:31:15.291309459Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:31:15.29131869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:31:15.291333613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:15.292411377Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T23:31:17.698024504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:31:18.625545539Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-04T23:31:19.372283758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57160"
level=warn ts=2026-05-04T23:31:20.290859309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:21.959859699Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T23:31:25.291624622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:25.292796476Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
ts=2026-05-04T23:31:25.424999086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-04T23:31:25.808640667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:31:25.814136621Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:31:28.625798367Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-04T23:31:30.290624201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:31:30.429029864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:31:30.433716061Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:31:31.008142492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:31:31.017533501Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:31:31.959995878Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-04T23:31:31.960020722Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=warn ts=2026-05-04T23:31:35.291463736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:35.292879773Z caller=checkpoint.go:372 msg="writing series" size="953 kB"
level=debug ts=2026-05-04T23:31:35.81600603Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:31:38.62576941Z caller=checkpoint.go:372 msg="writing series" size="774 kB"
level=debug ts=2026-05-04T23:31:38.625792949Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=warn ts=2026-05-04T23:31:40.290966251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:41.95893135Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-04T23:31:45.086610948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:31:45.095695027Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T23:31:45.270508417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53504"
level=info ts=2026-05-04T23:31:45.272849002Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:31:45.273215855Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:31:45.290576446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:31:45.290608002Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:31:45.290615708Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:31:45.290606384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:45.292669151Z caller=checkpoint.go:372 msg="writing series" size="991 B"
level=debug ts=2026-05-04T23:31:48.62577515Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-04T23:31:49.272699932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:31:49.283289185Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:31:50.290765487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:51.959556921Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-04T23:31:52.818087635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:31:55.290609906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:31:55.296879017Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-04T23:31:55.296912541Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=5
ts=2026-05-04T23:31:55.427053413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:31:58.62563369Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:32:00.291457651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:32:00.992576339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:32:01.002038338Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:32:01.652153935Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:32:01.959392082Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-05-04T23:32:05.290733466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:05.293106718Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=debug ts=2026-05-04T23:32:08.627542592Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-04T23:32:08.627566089Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=4
level=warn ts=2026-05-04T23:32:10.290693593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:11.959297617Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-04T23:32:15.291524742Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:32:15.291557328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:32:15.291565576Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:32:15.291646702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:15.292778445Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-04T23:32:18.625448337Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-04T23:32:20.291144054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:32:20.350327739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:32:20.357244154Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:32:21.959899444Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=info ts=2026-05-04T23:32:23.68498657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:32:23.692990304Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:32:25.291450857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:25.292556059Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T23:32:25.429661888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:32:26.257161276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:32:26.261250259Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:32:28.625637715Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-04T23:32:28.694716035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:32:30.291436897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:31.959165549Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-04T23:32:35.290728356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:35.293073802Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
level=debug ts=2026-05-04T23:32:35.293090908Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-04T23:32:36.419498945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:32:36.425875798Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:32:37.382137153Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:32:38.626202901Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-04T23:32:39.606515989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:32:39.615288036Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:32:39.615317341Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:32:39.615346021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:32:39.622357235Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:32:40.291113202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:41.959675738Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T23:32:43.121645881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="54 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:32:43.128986766Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:32:45.27251282Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:32:45.273224185Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:32:45.290534361Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:32:45.290570213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:32:45.290578883Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:32:45.290575421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:45.292889781Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-04T23:32:48.626317063Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-04T23:32:50.291012989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:51.958953062Z caller=checkpoint.go:372 msg="writing series" size="520 kB"
level=debug ts=2026-05-04T23:32:51.95899023Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-05-04T23:32:55.290609077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:32:55.292701704Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-04T23:32:55.296824744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:32:55.306420849Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
ts=2026-05-04T23:32:55.431960328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:32:58.62649941Z caller=checkpoint.go:372 msg="writing series" size="679 kB"
level=warn ts=2026-05-04T23:33:00.290877044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:01.959509605Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=warn ts=2026-05-04T23:33:05.290964734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:05.292145512Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=debug ts=2026-05-04T23:33:05.292162798Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-04T23:33:08.625520405Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T23:33:09.103561267Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:33:10.291546026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:11.96005659Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=info ts=2026-05-04T23:33:14.261707628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:33:14.267468465Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:33:14.297807761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:33:14.302069382Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:33:15.291283114Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:33:15.291323077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:33:15.291331523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:33:15.291341775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:15.292420808Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-04T23:33:18.610796828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:33:18.619248183Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:33:18.626319524Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=warn ts=2026-05-04T23:33:20.291056059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:21.959524299Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T23:33:22.487772449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:33:22.494782719Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:33:24.231532909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:33:24.233224953Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T23:33:25.290802079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:25.292996082Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-04T23:33:25.434143552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:33:25.754324495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:33:25.766669582Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:33:26.77769806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:33:26.784689372Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:33:28.625976825Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-04T23:33:28.626000483Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-04T23:33:30.290700431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:31.962845842Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-04T23:33:31.962886654Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=warn ts=2026-05-04T23:33:35.290931139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:35.292107001Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-04T23:33:38.625959511Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=warn ts=2026-05-04T23:33:40.290909501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:41.958831655Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-04T23:33:42.397570035Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:33:45.273116028Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:33:45.274326208Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:33:45.290542872Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:33:45.290572036Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:33:45.290586504Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:33:45.290612403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:45.29278754Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-04T23:33:48.62643843Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
ts=2026-05-04T23:33:49.383648356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38532"
level=warn ts=2026-05-04T23:33:50.291136245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:51.959373788Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T23:33:51.959400134Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-05-04T23:33:54.835527392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:56498"
level=info ts=2026-05-04T23:33:54.88920684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:33:54.896361629Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:33:54.896390196Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:33:54.896420003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:33:54.900773411Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:33:55.291431058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:33:55.292707514Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
ts=2026-05-04T23:33:55.436204944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:33:58.626389214Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
level=warn ts=2026-05-04T23:34:00.290880764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:34:00.780523646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:34:00.785104641Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:34:00.785128599Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:34:00.785166802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:34:00.788340114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:34:01.95942362Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-04T23:34:03.493383089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:34:03.500305662Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:34:05.291124078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:34:05.292214762Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-04T23:34:08.305330764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:34:08.312975069Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:34:08.627214524Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-04T23:34:08.627236278Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=debug ts=2026-05-04T23:34:09.059870352Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:34:10.290679588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:34:11.959545348Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-04T23:34:12.110103066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:34:12.118389265Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-bmfg\"}"
level=info ts=2026-05-04T23:34:12.972973302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:34:12.9768659Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:34:13.989018312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:34:13.993891215Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:34:15.290879835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:34:15.29087976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:34:15.290903322Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:34:15.290911477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:34:15.291993246Z caller=checkpoint.go:384 msg="flushing series" totalSize="107 kB" series=1
level=info ts=2026-05-04T23:34:15.294869383Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-04T23:34:15.296504169Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004589788s
level=debug ts=2026-05-04T23:34:19.27832679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:34:19.386059874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51304"
level=info ts=2026-05-04T23:34:20.066215014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:34:20.074395347Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:34:20.291690114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:34:24.836601741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42342"
level=warn ts=2026-05-04T23:34:25.291316646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:34:25.437663792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T23:34:30.291342262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:34:35.291197962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:34:36.015771704Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:34:40.29123356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:34:42.179561965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:34:42.185251796Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:34:43.09190947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:34:43.096506763Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:34:45.272342026Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:34:45.27340944Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T23:34:45.290664488Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T23:34:45.290691713Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:34:45.29069712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:34:45.29070837Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:34:45.290715913Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T23:34:45.290988234Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-04T23:34:45.290998148Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:34:45.291017364Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.695µs success=true
level=debug ts=2026-05-04T23:34:45.29124504Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T23:34:48.625310247Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-04T23:34:50.29092008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:34:51.9584965Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-04T23:34:53.463666031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:34:53.471737154Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:34:53.471767493Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:34:53.471793892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:34:53.477621077Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:34:54.785908099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:34:55.291388024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:34:55.291426714Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-04T23:34:55.439573492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:34:58.624930132Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-05-04T23:34:59.02030969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:34:59.027055426Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:34:59.027078456Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:34:59.02710289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:34:59.032806229Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:35:00.291071251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:01.958515292Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-04T23:35:04.586067854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:35:04.593280004Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:35:05.290668041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:05.291920895Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-04T23:35:06.892924091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:35:06.900256499Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:35:07.973396252Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:35:08.639777193Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T23:35:08.639801609Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=warn ts=2026-05-04T23:35:10.291310627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:11.965284665Z caller=checkpoint.go:372 msg="writing series" size="9.2 MB"
level=debug ts=2026-05-04T23:35:11.965314031Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.2 MB" series=1
level=info ts=2026-05-04T23:35:13.255891442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:35:13.262718563Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:35:13.306145078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:35:13.31205256Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T23:35:15.283195836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46056"
level=info ts=2026-05-04T23:35:15.29056078Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:35:15.290578195Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:35:15.290587551Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:35:15.290596467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:15.291702869Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-04T23:35:18.625555338Z caller=checkpoint.go:372 msg="writing series" size="637 kB"
level=warn ts=2026-05-04T23:35:20.291079105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:21.958542714Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-04T23:35:24.538836457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:35:24.544909839Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:35:25.291284342Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-04T23:35:25.291291067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:35:25.441073074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:35:28.62638408Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T23:35:28.62641582Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=warn ts=2026-05-04T23:35:30.291050469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:31.958213692Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=debug ts=2026-05-04T23:35:32.372768033Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:35:35.290976776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:35.292127749Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-05-04T23:35:37.824976809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:35:38.625088686Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:35:40.291228512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:35:41.458318201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:35:41.463873486Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:35:42.011728047Z caller=checkpoint.go:372 msg="writing series" size="56 MB"
level=debug ts=2026-05-04T23:35:42.011762065Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=4
level=info ts=2026-05-04T23:35:44.662315102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:35:44.669450188Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:35:45.272521274Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:35:45.272917287Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:35:45.284289088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49804"
level=info ts=2026-05-04T23:35:45.291009046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:35:45.291032837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:45.291064935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:35:45.291075417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:35:45.292233603Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-04T23:35:48.113669404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:35:48.121086781Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:35:48.625624513Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=warn ts=2026-05-04T23:35:50.291248197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:51.958715384Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=info ts=2026-05-04T23:35:55.084475348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:35:55.093365591Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:35:55.093392831Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:35:55.093421798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:35:55.101391474Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:35:55.291117664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:35:55.291384394Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-04T23:35:55.291397024Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-04T23:35:55.442043246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:35:56.621111584Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:35:58.625535506Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=warn ts=2026-05-04T23:36:00.291284898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:01.959155137Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-04T23:36:02.912667506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:36:02.921855566Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:36:05.290690203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:05.292452305Z caller=checkpoint.go:372 msg="writing series" size="496 kB"
level=info ts=2026-05-04T23:36:07.27082009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:36:07.277386182Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:36:08.207989883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:36:08.218284979Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:36:08.624883543Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-04T23:36:08.624919078Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-04T23:36:10.291361271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:11.958412707Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-04T23:36:15.290823476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:36:15.290843367Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:36:15.290865742Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:36:15.290873711Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:36:15.291982427Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T23:36:18.625242611Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-04T23:36:20.291242263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:20.738555671Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:36:21.958912733Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-04T23:36:24.843455105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37386"
level=warn ts=2026-05-04T23:36:25.291438996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:25.291520601Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
ts=2026-05-04T23:36:25.444509258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:36:28.625641725Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-04T23:36:28.970752852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:36:28.979520588Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T23:36:30.291338599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:30.965251457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:36:31.960244081Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-04T23:36:31.960326547Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=info ts=2026-05-04T23:36:34.665114832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:36:34.669418885Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:36:35.290923416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:35.292777609Z caller=checkpoint.go:372 msg="writing series" size="966 kB"
level=info ts=2026-05-04T23:36:35.803776696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:36:35.812253223Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:36:36.701149841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:36:36.708762147Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:36:38.626005857Z caller=checkpoint.go:372 msg="writing series" size="901 kB"
level=debug ts=2026-05-04T23:36:38.6260293Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=2
level=warn ts=2026-05-04T23:36:40.291630401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:41.958997073Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-04T23:36:42.936012931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:36:42.944328269Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-bmfg\"}"
level=info ts=2026-05-04T23:36:45.272703052Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:36:45.272713565Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:36:45.288113638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56166"
level=info ts=2026-05-04T23:36:45.290546313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:36:45.290571659Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:36:45.290580958Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:36:45.290580074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:45.291603691Z caller=checkpoint.go:372 msg="writing series" size="991 B"
level=debug ts=2026-05-04T23:36:48.625021027Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-04T23:36:50.291205177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:51.958345931Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-04T23:36:53.644486101Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T23:36:54.845196457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39464"
level=warn ts=2026-05-04T23:36:55.291439473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:36:55.295708585Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-04T23:36:55.295730564Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=5
ts=2026-05-04T23:36:55.445986852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:36:58.625184584Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:37:00.290652105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:37:01.958467947Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-04T23:37:02.83394465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:37:02.841067547Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:37:05.291596695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:37:05.291740567Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=debug ts=2026-05-04T23:37:08.627242736Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-04T23:37:08.627281601Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=warn ts=2026-05-04T23:37:10.29123411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:37:11.491314484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:37:11.498598197Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:37:11.959221512Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
ts=2026-05-04T23:37:15.28948943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53362"
level=info ts=2026-05-04T23:37:15.291092674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:37:15.291108332Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:37:15.291116078Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:37:15.291124918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:37:15.292288465Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-04T23:37:17.647776001Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:37:18.62505412Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-04T23:37:20.290872345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:37:21.487724184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:37:21.496249091Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:37:21.958567676Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=debug ts=2026-05-04T23:37:24.310408119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:37:24.312021457Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T23:37:24.639946898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:37:24.647343774Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:37:24.781917619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:37:24.786324787Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:37:25.291337223Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-04T23:37:25.291352609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:37:25.447334328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:37:28.625582416Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-04T23:37:30.290635356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:37:31.958399444Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-04T23:37:35.2907336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:37:35.292120321Z caller=checkpoint.go:372 msg="writing series" size="441 kB"
level=debug ts=2026-05-04T23:37:35.292136199Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-04T23:37:37.447524818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:37:37.45865807Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-bmfg\"}"
level=debug ts=2026-05-04T23:37:38.62541595Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-04T23:37:40.291341891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:37:41.959005988Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T23:37:42.895022849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:37:42.899371031Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:37:45.272612042Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:37:45.272943426Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:37:45.290620777Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:37:45.290659771Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:37:45.290670654Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:37:45.290663767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:37:45.292058005Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-04T23:37:48.625258698Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
ts=2026-05-04T23:37:49.40348264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51508"
level=debug ts=2026-05-04T23:37:49.763880319Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:37:50.291603598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:37:51.959062934Z caller=checkpoint.go:372 msg="writing series" size="600 kB"
level=debug ts=2026-05-04T23:37:51.959089336Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-04T23:37:52.632010546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:37:52.63942995Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T23:37:54.847523088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46408"
level=warn ts=2026-05-04T23:37:55.291616828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:37:55.291632115Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-04T23:37:55.449040514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:37:58.625401594Z caller=checkpoint.go:372 msg="writing series" size="586 kB"
level=warn ts=2026-05-04T23:38:00.290985349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:01.958476244Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=warn ts=2026-05-04T23:38:05.29119557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:05.291457094Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=debug ts=2026-05-04T23:38:05.291472503Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-04T23:38:05.339330915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:38:05.351722758Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:38:08.625049582Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T23:38:09.706681929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:38:10.291299879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:11.958822849Z caller=checkpoint.go:372 msg="writing series" size="781 kB"
level=info ts=2026-05-04T23:38:15.291371469Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:38:15.291402324Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:38:15.291410248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:38:15.291414882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:15.291449876Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-04T23:38:15.293640896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55996"
level=info ts=2026-05-04T23:38:15.768817903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:38:15.778711881Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:38:18.625587414Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=debug ts=2026-05-04T23:38:18.625630039Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-04T23:38:19.405626182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48868"
level=info ts=2026-05-04T23:38:19.705020394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:38:19.713564917Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-bmfg\"}"
level=warn ts=2026-05-04T23:38:20.291302189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:38:20.690887687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:38:20.695471371Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:38:21.958717999Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-04T23:38:24.230335708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:38:25.290712166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:25.291851368Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-04T23:38:25.450693211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:38:28.625648019Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
level=warn ts=2026-05-04T23:38:30.291634946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:31.9603228Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-04T23:38:31.960354664Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=4
level=info ts=2026-05-04T23:38:32.677776429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:38:32.68711617Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T23:38:35.290834832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:35.292038587Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-04T23:38:38.624866551Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-05-04T23:38:38.729225485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:38:38.734444244Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:38:40.290846958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:41.958880913Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-04T23:38:45.272604823Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:38:45.272939103Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-05-04T23:38:45.291328094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:38:45.291334474Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:38:45.291366054Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:38:45.291380127Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:38:45.291362597Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-04T23:38:48.625604466Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-04T23:38:49.407753634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:52908"
level=info ts=2026-05-04T23:38:49.827189685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:38:49.836228831Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:38:50.29151846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:50.583094464Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:38:51.958486956Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T23:38:51.958513842Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-04T23:38:55.290924675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:38:55.29216315Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
ts=2026-05-04T23:38:55.45224683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:38:58.625749516Z caller=checkpoint.go:372 msg="writing series" size="355 kB"
level=warn ts=2026-05-04T23:39:00.290722917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:39:01.958719592Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-04T23:39:05.048303824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:39:05.290910088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:39:05.292039191Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-04T23:39:06.258330018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b77f8ecd7f94ba6 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:39:06.266584291Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b77f8ecd7f94ba6/19df53738b0%3A19df543facf%3Abcfaa74b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-bmfg\"}"
level=error ts=2026-05-04T23:39:06.266609354Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:39:06.266636634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:39:06.270475491Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:39:08.626775835Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-04T23:39:08.626799469Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=info ts=2026-05-04T23:39:09.566397924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:39:09.576116859Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:39:10.291406424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:39:11.958021091Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-04T23:39:12.826591742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:39:12.830565486Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:39:14.684510494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:39:14.692125514Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:39:15.290840631Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:39:15.290885468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:39:15.290895727Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:39:15.290949373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:39:15.291988516Z caller=checkpoint.go:384 msg="flushing series" totalSize="132 kB" series=1
level=info ts=2026-05-04T23:39:15.298110212Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-04T23:39:15.300162452Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008985136s
level=debug ts=2026-05-04T23:39:15.631098822Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:39:20.291159043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:39:20.96414708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:39:20.969616116Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T23:39:25.290620878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:39:25.453359905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-04T23:39:29.834511387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:39:29.838396588Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:39:30.291064203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:39:35.291098239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:39:37.348444275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:39:37.356209346Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:39:40.291441762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:39:41.197256451Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:39:45.272531879Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:39:45.27253862Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:39:45.290826227Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T23:39:45.290810866Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:39:45.290855318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:39:45.290864918Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:39:45.290872098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:39:45.290977271Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-04T23:39:45.290986075Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:39:45.291003776Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.169µs success=true
level=debug ts=2026-05-04T23:39:45.291182508Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-04T23:39:45.298932509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47120"
level=debug ts=2026-05-04T23:39:47.76430021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:39:48.625112262Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
ts=2026-05-04T23:39:49.412349148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54482"
level=warn ts=2026-05-04T23:39:50.290859375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:39:51.958921788Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-04T23:39:53.406088169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:39:53.412953088Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:39:53.465008515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:39:53.469474523Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T23:39:55.290891611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:39:55.291993556Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-04T23:39:55.454836731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:39:58.625779804Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=warn ts=2026-05-04T23:40:00.291136216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:01.958127941Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-04T23:40:04.346490505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:40:04.355573089Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:40:04.567309485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:40:04.571545461Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:40:05.291031624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:05.29219271Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=debug ts=2026-05-04T23:40:08.639562314Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T23:40:08.639588493Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=warn ts=2026-05-04T23:40:10.2910885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:11.961419752Z caller=checkpoint.go:372 msg="writing series" size="9.9 MB"
level=debug ts=2026-05-04T23:40:11.961442661Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.9 MB" series=1
level=info ts=2026-05-04T23:40:15.291245265Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:40:15.291237569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:15.291276668Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-04T23:40:15.29128932Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:40:15.291297988Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:40:16.594089177Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:40:16.594136941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:40:16.603546458Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-04T23:40:18.625620061Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
ts=2026-05-04T23:40:19.414950189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42964"
level=warn ts=2026-05-04T23:40:20.291424398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:21.958309941Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-04T23:40:22.089825032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:40:22.094216987Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:40:22.094234938Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:40:22.094284626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:40:22.097526775Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:40:25.29113612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:25.291288529Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
ts=2026-05-04T23:40:25.456593313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:40:28.134300092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:40:28.141090154Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:40:28.626404485Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-04T23:40:28.626427226Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=5
level=info ts=2026-05-04T23:40:29.591091031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:40:29.59734614Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-04T23:40:30.291621545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:31.958590807Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-04T23:40:34.660402707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:40:34.663857963Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:40:35.254415027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:40:35.291462271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:35.291555934Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-05-04T23:40:38.625422993Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:40:40.290915562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:42.010239344Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-05-04T23:40:42.010288035Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=4
level=info ts=2026-05-04T23:40:43.791581365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:40:43.798840118Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:40:45.272623982Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:40:45.272829475Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:40:45.291558917Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:40:45.291578805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:40:45.29158663Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:40:45.291626293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:45.291649047Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-04T23:40:45.424211573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:40:45.431712375Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:40:45.431733372Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:40:45.431756925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:40:45.43872617Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:40:47.330397233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:40:47.338492641Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:40:48.33431634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:40:48.338198571Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:40:48.338217701Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:40:48.338240586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:40:48.341614519Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:40:48.626065083Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=debug ts=2026-05-04T23:40:50.004482199Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:40:50.290811849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:40:51.128632819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:40:51.135659007Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-04T23:40:51.959252031Z caller=checkpoint.go:372 msg="writing series" size="342 kB"
level=debug ts=2026-05-04T23:40:51.959351481Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-05-04T23:40:55.291213968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:40:55.291484611Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-04T23:40:55.458214576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:40:58.625558794Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=info ts=2026-05-04T23:41:00.195484556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:41:00.200028669Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:41:00.291574183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:01.959360159Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-04T23:41:02.069337282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:41:02.073857485Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:41:05.11435379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:41:05.12301328Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T23:41:05.291100738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:05.292447804Z caller=checkpoint.go:372 msg="writing series" size="399 kB"
level=debug ts=2026-05-04T23:41:05.292462102Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-04T23:41:08.624813546Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-04T23:41:10.290586626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:11.881324641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:41:11.883246561Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:41:11.958477272Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-05-04T23:41:15.291338766Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:41:15.291350761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:15.291373366Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:41:15.291382632Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:41:15.291392461Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-04T23:41:15.303489557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38014"
level=debug ts=2026-05-04T23:41:18.625420354Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-04T23:41:20.290872933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:21.959140124Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-04T23:41:23.62455884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:41:23.634284124Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:41:23.634320886Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:41:23.634349178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:41:23.641360482Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:41:25.290841861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:25.29198588Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
ts=2026-05-04T23:41:25.459338639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:41:25.779425456Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:41:27.452609146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:41:27.460959067Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:41:27.944311134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:41:27.948672038Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:41:28.624921598Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-04T23:41:30.291608155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:31.959738995Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-04T23:41:31.959763593Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=8
level=info ts=2026-05-04T23:41:32.468549258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:41:32.47536554Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:41:34.900372297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:41:34.908396028Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:41:35.290688892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:35.292731644Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T23:41:35.29274928Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-04T23:41:35.814076367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:41:35.82231379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:41:38.625483526Z caller=checkpoint.go:372 msg="writing series" size="793 kB"
level=warn ts=2026-05-04T23:41:40.290886366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:41.958777706Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=info ts=2026-05-04T23:41:41.964831516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:41:41.970238163Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:41:44.010985266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:41:44.015523999Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:41:45.272307067Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:41:45.273350135Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T23:41:45.290621004Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:41:45.290645553Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:41:45.290661138Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:41:45.290659673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:45.291864253Z caller=checkpoint.go:372 msg="writing series" size="991 B"
level=debug ts=2026-05-04T23:41:48.625538295Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=debug ts=2026-05-04T23:41:49.584415427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:41:50.291547859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:51.958302818Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-04T23:41:55.219120632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:41:55.229532659Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:41:55.290574449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:41:55.297162741Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-05-04T23:41:55.297183979Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.8 MB" series=6
ts=2026-05-04T23:41:55.460665971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:41:56.746235263Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:41:56.746308541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:41:56.752023059Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:41:58.625066617Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:42:00.291324965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:01.958013236Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-04T23:42:05.291419273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:05.291492891Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-04T23:42:08.116498468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:42:08.127822137Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:42:08.627628138Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-04T23:42:08.627652705Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=4
level=warn ts=2026-05-04T23:42:10.29132673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:11.958654063Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-04T23:42:15.290967264Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:42:15.290972309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:15.291011312Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:42:15.291031466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:42:15.292236469Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T23:42:18.153602747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:42:18.161221502Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:42:18.62535712Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
ts=2026-05-04T23:42:19.424523402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34610"
level=info ts=2026-05-04T23:42:19.967475959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:42:19.976078373Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:42:19.976104627Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:42:19.976134173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:42:19.984681312Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:42:20.290743351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:21.958574983Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-05-04T23:42:25.046395852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:42:25.051122Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:42:25.291426377Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-04T23:42:25.291431022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:42:25.462568389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:42:26.922835714Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:42:27.687595686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:42:27.692282121Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:42:28.023013536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:42:28.029732809Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-04T23:42:28.625583062Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-04T23:42:30.291547014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:42:31.048575786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:42:31.05605536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:42:31.947931058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:42:31.953163562Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:42:31.958241273Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-04T23:42:35.290722516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:35.292215561Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=debug ts=2026-05-04T23:42:35.292236811Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=debug ts=2026-05-04T23:42:37.19079862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:42:38.154796443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:42:38.161011221Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:42:38.625422219Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=warn ts=2026-05-04T23:42:40.291450105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:41.958483553Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T23:42:45.272784882Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:42:45.273172532Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:42:45.291542037Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:42:45.291583359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:42:45.291603086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:42:45.29167128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:45.291896439Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-04T23:42:48.625108664Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
ts=2026-05-04T23:42:49.426112056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35816"
level=info ts=2026-05-04T23:42:49.80633014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:42:49.815484866Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:42:50.290647858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:51.958597154Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T23:42:51.958621269Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
ts=2026-05-04T23:42:54.863890686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35718"
level=warn ts=2026-05-04T23:42:55.290991257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:42:55.292029454Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-04T23:42:55.464806662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:42:58.625772688Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=debug ts=2026-05-04T23:42:59.610392134Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:43:00.290857095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:01.959036291Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=warn ts=2026-05-04T23:43:05.290661824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:05.291771388Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=debug ts=2026-05-04T23:43:05.2917897Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-04T23:43:07.841124206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:43:07.850527141Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:43:08.624913741Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-04T23:43:10.074732368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:43:10.081060363Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:43:10.291570559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:43:10.862703828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:43:10.870768112Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:43:11.959310157Z caller=checkpoint.go:372 msg="writing series" size="862 kB"
level=debug ts=2026-05-04T23:43:11.959338046Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-04T23:43:15.290672692Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:43:15.290711259Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:43:15.290722458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:43:15.290730588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:15.291850231Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-04T23:43:18.235459725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:43:18.625333735Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=warn ts=2026-05-04T23:43:20.291187625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:21.958420986Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T23:43:22.559878711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:43:22.565958232Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:43:23.265626333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:43:23.273000876Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
ts=2026-05-04T23:43:24.865192623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41486"
level=info ts=2026-05-04T23:43:25.19596075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:43:25.202296426Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T23:43:25.291339143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:25.291410944Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-04T23:43:25.466878502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-04T23:43:25.608100984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:43:25.613046488Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:43:28.624863613Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-04T23:43:30.291082953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:31.961396009Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-04T23:43:31.961422916Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=6
level=debug ts=2026-05-04T23:43:33.909408816Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:43:35.290766015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:35.291967859Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-04T23:43:38.625332791Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=warn ts=2026-05-04T23:43:40.290809709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:41.959030102Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-04T23:43:45.272847012Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:43:45.273765818Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:43:45.291113053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:43:45.291131953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:45.291154426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:43:45.291176363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:43:45.291196583Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
ts=2026-05-04T23:43:45.311998872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57408"
level=info ts=2026-05-04T23:43:45.516574267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:43:45.524641846Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:43:48.624974698Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=info ts=2026-05-04T23:43:49.396769008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:43:49.403253707Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T23:43:49.430474316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41380"
level=warn ts=2026-05-04T23:43:50.291395256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:51.958766777Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T23:43:51.958792196Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-04T23:43:54.652215012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:43:54.656453588Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-04T23:43:54.866905148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55556"
level=warn ts=2026-05-04T23:43:55.290990136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:43:55.292222634Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=debug ts=2026-05-04T23:43:55.446743142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:43:55.46892525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:43:56.746479303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:43:56.754342725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:43:58.625413401Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=warn ts=2026-05-04T23:44:00.29100006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:44:01.958714176Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-04T23:44:03.945859698Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:44:04.784679709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:44:04.788923211Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:44:05.290855936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:44:05.292033171Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-04T23:44:08.625794019Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-04T23:44:08.625816103Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=5
level=warn ts=2026-05-04T23:44:10.291315164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:44:11.958914027Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-04T23:44:14.356421821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:44:14.362587392Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T23:44:15.291457153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:44:15.291451306Z caller=checkpoint.go:384 msg="flushing series" totalSize="152 kB" series=1
level=debug ts=2026-05-04T23:44:15.291486732Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:44:15.291496503Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:44:15.29149271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:44:15.299356811Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-04T23:44:15.301025761Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009887467s
level=info ts=2026-05-04T23:44:18.804616994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:44:18.813057303Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
ts=2026-05-04T23:44:19.43219596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41200"
level=info ts=2026-05-04T23:44:19.725344368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:44:19.73084283Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T23:44:20.291621245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T23:44:25.291563237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:44:25.469997079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=warn ts=2026-05-04T23:44:30.291352133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:44:30.523927849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:44:30.533868629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:44:31.030444182Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:44:32.549341763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:44:32.557368775Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:44:35.291196421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:44:38.923412864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:44:38.931824193Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:44:40.291103837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:44:45.272598524Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:44:45.273139167Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:44:45.290518408Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T23:44:45.290559134Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:44:45.290587916Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:44:45.290608618Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:44:45.290594453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:44:45.290733736Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-04T23:44:45.290746137Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:44:45.290774663Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=21.581µs success=true
level=debug ts=2026-05-04T23:44:45.291051952Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T23:44:48.624681499Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
ts=2026-05-04T23:44:49.434795211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34042"
level=warn ts=2026-05-04T23:44:50.291054563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:44:51.169487901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:44:51.176976786Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:44:51.958400096Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-04T23:44:55.291215382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:44:55.291279339Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-04T23:44:55.471222136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:44:56.843445498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:44:56.845361911Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:44:58.287305269Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:44:58.625190069Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=warn ts=2026-05-04T23:45:00.291422354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:45:00.614023072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:45:00.620757195Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:45:00.661877756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:45:00.667699154Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:45:01.957769249Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-04T23:45:02.503630841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:45:02.5077201Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:45:03.462419209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:45:03.467573336Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T23:45:05.290627766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:05.291757459Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=debug ts=2026-05-04T23:45:08.640892651Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T23:45:08.640934353Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=warn ts=2026-05-04T23:45:10.291540456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:11.966992461Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-04T23:45:11.967020737Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=1
level=info ts=2026-05-04T23:45:15.290811528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:45:15.290852363Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:45:15.290865268Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:45:15.290886852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:15.291985582Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-05-04T23:45:15.317013121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:39340"
level=info ts=2026-05-04T23:45:17.954513012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:45:17.964058589Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:45:18.625841006Z caller=checkpoint.go:372 msg="writing series" size="632 kB"
level=warn ts=2026-05-04T23:45:20.291123978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:21.958163332Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-04T23:45:23.342640675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:45:23.349502743Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T23:45:24.87234958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41628"
level=warn ts=2026-05-04T23:45:25.291330106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:25.291395108Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
ts=2026-05-04T23:45:25.473011141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:45:28.626516343Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-04T23:45:28.626544167Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=5
level=debug ts=2026-05-04T23:45:29.517596856Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:45:30.29068765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:31.958228054Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-04T23:45:34.049538445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:45:34.0602772Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:45:35.290712518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:35.291932592Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-05-04T23:45:38.624443041Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:45:40.29156369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:42.011962656Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-04T23:45:42.012018612Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=4
level=info ts=2026-05-04T23:45:42.736439419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:45:42.744433915Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:45:45.272539306Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:45:45.272649293Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-04T23:45:45.273951194Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-04T23:45:45.273993107Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975486 n=1
level=debug ts=2026-05-04T23:45:45.274035261Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975486 n=1
level=info ts=2026-05-04T23:45:45.290983798Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:45:45.291010864Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:45:45.291019223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:45:45.291040445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:45.291115692Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-04T23:45:47.455520354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:45:47.461421863Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-04T23:45:47.461461077Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:45:47.461488969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:45:47.467697731Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:45:47.997693172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:45:48.625898071Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=warn ts=2026-05-04T23:45:50.291117639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:45:50.462872394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:45:50.467294744Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:45:51.958321853Z caller=checkpoint.go:372 msg="writing series" size="342 kB"
level=debug ts=2026-05-04T23:45:51.958345097Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-04T23:45:53.610517208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:45:53.617682394Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T23:45:55.291331802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:45:55.291665215Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-04T23:45:55.47483014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:45:56.268242755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:45:56.272511575Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:45:58.624865328Z caller=checkpoint.go:372 msg="writing series" size="624 B"
level=warn ts=2026-05-04T23:46:00.290789175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:01.959636845Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-04T23:46:02.567864157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:46:02.577290604Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:46:02.980961374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:46:02.986886679Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T23:46:05.290770296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:05.292532931Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
level=debug ts=2026-05-04T23:46:05.292557967Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=debug ts=2026-05-04T23:46:05.432339289Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:46:08.62525302Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-04T23:46:10.291305894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:11.957941717Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-05-04T23:46:15.291433929Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:46:15.291464686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:15.291492384Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:46:15.291501711Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:46:15.291534209Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-04T23:46:15.32139081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34274"
level=info ts=2026-05-04T23:46:17.012590485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:46:17.025015275Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:46:18.625217506Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=info ts=2026-05-04T23:46:19.364699567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:46:19.373212621Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T23:46:19.441562434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51452"
level=warn ts=2026-05-04T23:46:20.291323414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:21.95862662Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T23:46:25.291391329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:25.291654855Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
ts=2026-05-04T23:46:25.476152123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-04T23:46:26.594299132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:46:26.600675045Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:46:28.625063359Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-04T23:46:30.290842213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:31.959106962Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-04T23:46:31.959132452Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=8
level=warn ts=2026-05-04T23:46:35.291154699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:35.292018699Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T23:46:35.292036605Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=debug ts=2026-05-04T23:46:36.744219456Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:46:38.625076853Z caller=checkpoint.go:372 msg="writing series" size="801 kB"
level=warn ts=2026-05-04T23:46:40.290726822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:46:41.174045737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:46:41.181180163Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:46:41.181210162Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:46:41.181256616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:46:41.184826186Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:46:41.643083394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:46:41.650934611Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:46:41.6509611Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:46:41.650989552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:46:41.658159714Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-04T23:46:41.958281548Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-04T23:46:45.272755713Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:46:45.273209359Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:46:45.290535512Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:46:45.290571874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:46:45.290586523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:46:45.290620751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:45.291658388Z caller=checkpoint.go:372 msg="writing series" size="991 B"
ts=2026-05-04T23:46:45.323321853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36724"
level=info ts=2026-05-04T23:46:46.475118481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:46:46.481532424Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T23:46:46.693544073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:46:46.697673789Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:46:48.045172902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:46:48.05286051Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:46:48.625463067Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=debug ts=2026-05-04T23:46:48.9246855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:46:49.443505684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33068"
level=warn ts=2026-05-04T23:46:50.290603288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:51.958626273Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-04T23:46:55.290825438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:46:55.296002058Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-05-04T23:46:55.296023755Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.9 MB" series=6
ts=2026-05-04T23:46:55.478087673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-04T23:46:57.647165367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:46:57.65167945Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:46:58.624887389Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:47:00.290947149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:47:00.753867346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:47:00.761224228Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:47:01.016120402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:47:01.02373509Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:47:01.023764772Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:47:01.023795039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:47:01.030798902Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:47:01.957840161Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-04T23:47:05.291176476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:47:05.291380588Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-04T23:47:07.594290059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:47:07.59954962Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:47:08.627730221Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-04T23:47:08.627756005Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=4
level=debug ts=2026-05-04T23:47:09.880879673Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:47:10.291501462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:47:11.958672132Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-04T23:47:14.096054609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:47:14.106279971Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:47:15.290848246Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:47:15.2908813Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T23:47:15.290877518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:47:15.290889837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:47:15.292174987Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-04T23:47:15.324785994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45422"
level=debug ts=2026-05-04T23:47:18.625205653Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T23:47:20.032495455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:47:20.039863752Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T23:47:20.294358005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:47:20.318346596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:47:20.322295347Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:47:20.789565449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:47:20.798313667Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-04T23:47:21.958505809Z caller=checkpoint.go:372 msg="writing series" size="293 kB"
level=warn ts=2026-05-04T23:47:25.291410899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:47:25.291463968Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T23:47:25.480299837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:47:28.625414293Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-04T23:47:30.290627851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:47:30.705948032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:47:31.958201897Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-04T23:47:33.277593906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:47:33.286089607Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T23:47:35.290909199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:47:35.292333504Z caller=checkpoint.go:372 msg="writing series" size="434 kB"
level=debug ts=2026-05-04T23:47:35.292349837Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=debug ts=2026-05-04T23:47:38.625422526Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-04T23:47:39.58571072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:47:39.59446255Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:47:39.59450145Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:47:39.594550979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:47:39.603313966Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:47:40.291368666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:47:41.68483704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:47:41.68931782Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:47:41.68934811Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:47:41.689376577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:47:41.693753139Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:47:41.958480638Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-04T23:47:44.372293857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:47:45.272556351Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:47:45.272887457Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:47:45.291285654Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:47:45.29131293Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:47:45.291321122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:47:45.291351219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:47:45.291642259Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-04T23:47:48.625098987Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-04T23:47:50.186024421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:47:50.194660529Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:47:50.291009147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:47:51.958664363Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T23:47:51.958689524Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-04T23:47:52.325953179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:47:52.330497467Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:47:53.959519686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:47:53.966833806Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:47:54.214412252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:47:54.220141365Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-04T23:47:54.880897811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34060"
level=warn ts=2026-05-04T23:47:55.290925146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:47:55.291104624Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-04T23:47:55.482359183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:47:58.625804214Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=warn ts=2026-05-04T23:48:00.2906495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:48:01.368597147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:48:01.373653941Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:48:01.958473206Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=warn ts=2026-05-04T23:48:05.291022076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:48:05.291086242Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-04T23:48:08.625533878Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T23:48:08.625562332Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-04T23:48:10.291011078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:48:11.532738721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:48:11.541348083Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:48:11.959339659Z caller=checkpoint.go:372 msg="writing series" size="939 kB"
level=info ts=2026-05-04T23:48:15.291453244Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:48:15.291488526Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:48:15.2915002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:48:15.291537059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:48:15.291546198Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-04T23:48:15.328685312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33472"
level=debug ts=2026-05-04T23:48:16.073834005Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:48:16.493120981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:48:16.500702089Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:48:18.047171059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:48:18.053579002Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:48:18.05361339Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:48:18.053640786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:48:18.05982525Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:48:18.624999429Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=debug ts=2026-05-04T23:48:20.146841251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:48:20.291399592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:48:21.958711603Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-04T23:48:24.882812998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51206"
level=info ts=2026-05-04T23:48:24.900253207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:48:24.907817894Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:48:25.291413323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:48:25.291460883Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
ts=2026-05-04T23:48:25.483857978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:48:28.148324762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:48:28.152543602Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:48:28.240792821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:48:28.246899446Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:48:28.62555584Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=debug ts=2026-05-04T23:48:28.625591156Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-04T23:48:30.291296348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:48:31.960440429Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-04T23:48:31.960476995Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=1
level=warn ts=2026-05-04T23:48:35.291588241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:48:35.291721824Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-04T23:48:38.624809862Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=warn ts=2026-05-04T23:48:40.291547786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:48:41.313663142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:48:41.323551928Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:48:41.958244474Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-04T23:48:43.513774405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:48:43.518214234Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:48:45.272579085Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:48:45.272786304Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-05-04T23:48:45.291217326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:48:45.291233156Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:48:45.291243446Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-04T23:48:45.291257471Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:48:45.291301048Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-04T23:48:45.330097836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55802"
level=debug ts=2026-05-04T23:48:47.204140877Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:48:48.625080537Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
ts=2026-05-04T23:48:49.451972723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40350"
level=warn ts=2026-05-04T23:48:50.291254991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:48:51.677383101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:48:51.683304631Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:48:51.958300184Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T23:48:51.958321442Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-04T23:48:53.504107385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:48:53.509900203Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-04T23:48:54.884371332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37622"
level=warn ts=2026-05-04T23:48:55.291311525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:48:55.291543301Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
ts=2026-05-04T23:48:55.486332721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:48:55.509475511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:48:55.518411062Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:48:55.518441171Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:48:55.518482459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:48:55.527586523Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:48:58.625116595Z caller=checkpoint.go:372 msg="writing series" size="394 kB"
level=warn ts=2026-05-04T23:49:00.290620795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:49:01.958713133Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-04T23:49:05.291289061Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-04T23:49:05.291325914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:49:08.626636584Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-04T23:49:08.626658886Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=5
level=info ts=2026-05-04T23:49:09.062519763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:49:09.073554847Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-04T23:49:10.291031407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:49:11.129550556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:49:11.138164904Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:49:11.958064672Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=debug ts=2026-05-04T23:49:12.059385932Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:49:15.290802964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:49:15.290811282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:49:15.290840701Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:49:15.290849443Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:49:15.291925944Z caller=checkpoint.go:384 msg="flushing series" totalSize="162 kB" series=1
level=info ts=2026-05-04T23:49:15.301590092Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-04T23:49:15.302885133Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011895425s
level=info ts=2026-05-04T23:49:15.687995484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:49:15.696780473Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:49:20.291178799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:49:21.818041938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:49:21.819647163Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T23:49:25.290987241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:49:25.487749691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:49:25.78467983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:49:25.789303195Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:49:28.013711389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:49:28.018040581Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:49:30.291474938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:49:34.695027386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:49:34.703963897Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:49:34.765060069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:49:34.774037076Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:49:35.290715295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:49:38.666922313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:49:38.672755191Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T23:49:40.2915021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:49:44.801225027Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:49:45.272569895Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:49:45.272973267Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:49:45.291382184Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:49:45.291417859Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:49:45.291428298Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T23:49:45.291442486Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-04T23:49:45.29154057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:49:45.291607771Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-04T23:49:45.291617717Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:49:45.291635856Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.398µs success=true
level=debug ts=2026-05-04T23:49:45.291895158Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T23:49:48.626566441Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-05-04T23:49:49.030531702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:49:49.037382109Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-04T23:49:49.455787222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:32942"
level=warn ts=2026-05-04T23:49:50.291255752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:49:51.959395345Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-04T23:49:55.290964418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:49:55.292100072Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-04T23:49:55.489226096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:49:58.625830584Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=warn ts=2026-05-04T23:50:00.291620513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:01.959509387Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-04T23:50:04.638530769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:50:04.645913917Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:50:04.87374698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:50:04.881101539Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:50:05.291320158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:05.292506423Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-05-04T23:50:08.635117154Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T23:50:08.635140734Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=8
level=warn ts=2026-05-04T23:50:10.291073235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:11.962736002Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-04T23:50:11.962757917Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=1
level=info ts=2026-05-04T23:50:15.291099464Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:50:15.291109758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:15.291130862Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:50:15.291140486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:50:15.292252722Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-04T23:50:15.993113884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:50:18.626115338Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=info ts=2026-05-04T23:50:19.310854906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:50:19.319007718Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T23:50:19.457915566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38300"
level=warn ts=2026-05-04T23:50:20.291470081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:50:20.54804862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:50:20.553496084Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:50:20.649571139Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:50:21.958863066Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-04T23:50:23.328277345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:50:23.332020274Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-04T23:50:24.889787253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39210"
level=warn ts=2026-05-04T23:50:25.2914614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:25.292587175Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-04T23:50:25.319693958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:50:25.32388175Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-04T23:50:25.49088351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:50:28.628110078Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-04T23:50:28.628170132Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=5
level=warn ts=2026-05-04T23:50:30.291020055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:50:30.936512754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:50:30.94218334Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:50:31.675442449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:50:31.682646829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:50:31.959255493Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-04T23:50:35.290774893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:35.292965776Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=debug ts=2026-05-04T23:50:38.62564707Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:50:40.290896649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:42.014317835Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-05-04T23:50:42.014352627Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=4
level=info ts=2026-05-04T23:50:45.273165355Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:50:45.273738048Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:50:45.290962228Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:50:45.290979636Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:50:45.290989035Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:50:45.291014095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:45.292235613Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-04T23:50:45.685765505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:50:45.694525325Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:50:47.760167383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T23:50:47.760219986Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-04T23:50:47.763688892Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T23:50:48.060395751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:50:48.064476099Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T23:50:48.360485345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="30 kB" avg_uncomp="30 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:50:48.363942056Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T23:50:48.626547175Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=info ts=2026-05-04T23:50:48.660961141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:50:48.664744306Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T23:50:48.960601034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:50:48.964011173Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T23:50:49.260723423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:50:49.263994168Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-04T23:50:49.459563647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39236"
level=info ts=2026-05-04T23:50:49.560199401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:50:49.564481257Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T23:50:50.290838348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:51.958820755Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-04T23:50:55.290978755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:50:55.292247566Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-04T23:50:55.292278896Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-04T23:50:55.492152345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:50:58.616461776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:50:58.619851015Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T23:50:58.625919811Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-04T23:50:59.813158219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:50:59.820821621Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:50:59.973385624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:50:59.977084966Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T23:51:00.069313719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:51:00.072796645Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T23:51:00.291008925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:51:00.877709006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:51:00.880781978Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T23:51:01.959484004Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-04T23:51:02.104450292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:51:02.108656169Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T23:51:02.338378623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:51:02.342050922Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T23:51:03.007486652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:51:03.011081584Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T23:51:05.200205111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:51:05.207422335Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T23:51:05.29068531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:51:05.293098068Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=debug ts=2026-05-04T23:51:08.018359583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:51:08.626225327Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-04T23:51:08.626249593Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-04T23:51:10.291027681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:51:11.494125768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:51:11.498375819Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:51:11.959653485Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-04T23:51:15.290883339Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:51:15.29089916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:51:15.290914849Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:51:15.290925191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:51:15.292073177Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-04T23:51:16.517310407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:51:16.523347547Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T23:51:16.899333413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:51:16.903235689Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:51:18.080745805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:51:18.086111911Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:51:18.626372348Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-05-04T23:51:18.758804628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:51:20.290596868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:51:21.959249079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:51:21.959569514Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=error ts=2026-05-04T23:51:21.96731197Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:51:25.122789185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:51:25.126219802Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-04T23:51:25.291444673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:51:25.292755272Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
ts=2026-05-04T23:51:25.493968485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:51:26.864067175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:51:26.867660489Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T23:51:27.197934334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:51:27.201340706Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T23:51:27.308841376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:51:27.311882626Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T23:51:28.145191309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:51:28.14848792Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T23:51:28.625710307Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-04T23:51:30.291011612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:51:31.960114334Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-04T23:51:31.960137829Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=info ts=2026-05-04T23:51:32.150634013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:51:32.154618847Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T23:51:34.848609047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:51:34.855393701Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-04T23:51:35.290723134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:51:35.293713768Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T23:51:35.293728423Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=debug ts=2026-05-04T23:51:38.626555573Z caller=checkpoint.go:372 msg="writing series" size="860 kB"
level=warn ts=2026-05-04T23:51:40.290842362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:51:41.329818915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:51:41.333527759Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T23:51:41.959472733Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-04T23:51:45.016009073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:51:45.023987871Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:51:45.27248172Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:51:45.27251103Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T23:51:45.290767044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:51:45.290784167Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:51:45.29079391Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:51:45.290784573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:51:45.292936561Z caller=checkpoint.go:372 msg="writing series" size="991 B"
ts=2026-05-04T23:51:45.341046356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:32812"
level=debug ts=2026-05-04T23:51:46.222707565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:51:48.625930602Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-04T23:51:50.291462489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:51:51.958828012Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-04T23:51:53.330493505Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:51:53.3305097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="123 kB" avg_uncomp="123 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:51:53.334885493Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T23:51:53.63158108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:51:53.635025353Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-04T23:51:54.268861157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:51:54.275451638Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T23:51:55.291624992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:51:55.296892092Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-05-04T23:51:55.296919556Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.0 MB" series=6
ts=2026-05-04T23:51:55.495290462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:51:58.625704004Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:52:00.290855822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:52:01.307871476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:52:01.31313406Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-04T23:52:01.313157945Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:52:01.313179693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:52:01.317484502Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:52:01.520004976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:52:01.524531947Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:52:01.959705986Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-04T23:52:03.639705057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:52:03.646683159Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T23:52:04.212393421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:52:04.215735643Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T23:52:05.291517803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:52:05.292659681Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-05-04T23:52:08.627771434Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-04T23:52:08.627851911Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=4
level=warn ts=2026-05-04T23:52:10.29120297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:52:10.54890081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:52:10.552737946Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T23:52:11.088411536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:52:11.09197266Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T23:52:11.959482196Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-04T23:52:12.19097712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:52:12.194768497Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T23:52:15.290888555Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:52:15.290916159Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:52:15.290923725Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:52:15.29094446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:52:15.292068743Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
ts=2026-05-04T23:52:15.342319636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37790"
level=info ts=2026-05-04T23:52:16.592348645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:52:16.596205377Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:52:16.878101587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:52:16.883188271Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:52:18.319448887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:52:18.326745051Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:52:18.62618123Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T23:52:19.635912672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:52:19.639302773Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T23:52:20.291051607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:52:20.777445431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:52:20.780972737Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T23:52:21.23592565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:52:21.239235175Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T23:52:21.346708041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:52:21.350041179Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T23:52:21.95889651Z caller=checkpoint.go:372 msg="writing series" size="358 kB"
level=info ts=2026-05-04T23:52:24.835706292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:52:24.839821072Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-04T23:52:24.896716318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33050"
level=warn ts=2026-05-04T23:52:25.291142575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:52:25.292272043Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T23:52:25.496826378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:52:28.090076581Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T23:52:28.625982534Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-04T23:52:30.290870103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:52:31.95888652Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-04T23:52:33.465941641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:52:33.472646073Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-04T23:52:35.290596831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:52:35.293256933Z caller=checkpoint.go:372 msg="writing series" size="547 kB"
level=debug ts=2026-05-04T23:52:35.293295315Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-05-04T23:52:36.953195376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:52:36.959775888Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:52:38.625773621Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-04T23:52:39.695180908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T23:52:40.291586001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:52:40.618765296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:52:40.622394661Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-04T23:52:41.959599742Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T23:52:43.114659216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:52:43.119292709Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:52:43.375574994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:52:43.378635951Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T23:52:44.579115136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:52:44.585580086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:52:45.272566654Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:52:45.272972115Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:52:45.291384965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T23:52:45.291394277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:52:45.291408386Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:52:45.291416577Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:52:45.293002478Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
ts=2026-05-04T23:52:45.343796253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51718"
level=debug ts=2026-05-04T23:52:48.625787898Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-04T23:52:50.291389364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:52:51.959590691Z caller=checkpoint.go:372 msg="writing series" size="459 kB"
level=debug ts=2026-05-04T23:52:51.9596174Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-04T23:52:53.269238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:52:53.273720513Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:52:54.108018731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:52:54.111152342Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T23:52:54.79606502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:52:54.801910625Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T23:52:55.291512128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:52:55.292553245Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-04T23:52:55.497957521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-04T23:52:57.789562023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:52:57.793478976Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T23:52:58.026010002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:52:58.029639576Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:52:58.029658175Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:52:58.029695275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:52:58.032664556Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:52:58.625798778Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=warn ts=2026-05-04T23:53:00.291614783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:53:01.959086469Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=debug ts=2026-05-04T23:53:03.632868782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:53:05.290955833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:53:05.292138071Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-05-04T23:53:08.625645134Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T23:53:08.625676581Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-04T23:53:10.291380847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:53:11.959404142Z caller=checkpoint.go:372 msg="writing series" size="850 kB"
level=info ts=2026-05-04T23:53:13.752221219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:53:13.757775765Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T23:53:13.847060838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:53:13.850848754Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T23:53:14.532903695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:53:14.536629105Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T23:53:14.82645009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:53:14.83078648Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:53:15.29127298Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:53:15.291296548Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T23:53:15.29128804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:53:15.291306386Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:53:15.292391736Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-04T23:53:15.34468833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48022"
level=info ts=2026-05-04T23:53:16.68605305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:53:16.692724486Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:53:16.692748109Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:53:16.692781109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:53:16.698454993Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:53:17.163795155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:53:17.167683549Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T23:53:17.398794893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:53:17.407041253Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T23:53:18.625718583Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
ts=2026-05-04T23:53:19.470653767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57408"
level=warn ts=2026-05-04T23:53:20.291240968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:53:20.636222566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:53:20.639801953Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T23:53:21.958625286Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-04T23:53:25.291554096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:53:25.292668837Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-04T23:53:25.499505812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:53:25.844708806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:53:25.84934743Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:53:26.316104683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:53:26.317973284Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T23:53:27.361961755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:53:27.368032323Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-04T23:53:28.625512989Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=debug ts=2026-05-04T23:53:28.62553571Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-04T23:53:30.291400361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:53:30.629725811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:53:30.637248005Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:53:31.961580756Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-04T23:53:31.961607039Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=1
level=info ts=2026-05-04T23:53:34.396734144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:53:34.400425988Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T23:53:34.839470046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:53:34.843251037Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:53:35.291584162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:53:35.292795551Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-04T23:53:35.308778434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:53:35.315979876Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:53:35.315996557Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:53:35.31601985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:53:35.322287378Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:53:36.693161243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:53:37.675092427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:53:37.678413231Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-04T23:53:38.625377649Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=warn ts=2026-05-04T23:53:40.2912506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:53:40.833143267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:53:40.837419857Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:53:41.958757957Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-04T23:53:45.272533846Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:53:45.273168718Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-05-04T23:53:45.291517808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:53:45.291527366Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:53:45.291546717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:53:45.291554611Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:53:45.292632869Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
ts=2026-05-04T23:53:45.345928697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51604"
level=info ts=2026-05-04T23:53:46.760234727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:53:46.767807652Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:53:48.625521713Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-04T23:53:48.997424699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="804 B" avg_comp="804 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:53:49.000822192Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T23:53:50.291504362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:53:50.360612159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:53:50.367115567Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:53:50.7597652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:53:50.762995306Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T23:53:51.959880856Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T23:53:51.959909336Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-05-04T23:53:54.901792298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59734"
level=warn ts=2026-05-04T23:53:55.291229142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:53:55.292456819Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
ts=2026-05-04T23:53:55.501371068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:53:58.626140088Z caller=checkpoint.go:372 msg="writing series" size="541 kB"
level=info ts=2026-05-04T23:54:00.200794773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:54:00.204091892Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T23:54:00.291427017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:54:00.888823192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:54:00.896801529Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-04T23:54:01.958861394Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-04T23:54:02.776309781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:54:02.779597208Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-04T23:54:03.076551171Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:54:05.290713207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:54:05.291928577Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-04T23:54:07.488117401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:54:08.254890212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:54:08.263664691Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:54:08.551721432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:54:08.554765469Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T23:54:08.626886146Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-04T23:54:08.626909015Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=5
level=warn ts=2026-05-04T23:54:10.290615455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:54:11.958890225Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-04T23:54:11.987035996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:54:11.990539254Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T23:54:12.289505406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:54:12.293894469Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:54:12.7610589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:54:12.764497766Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T23:54:15.2913689Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:54:15.291398232Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:54:15.291407446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:54:15.29139671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:54:15.292488624Z caller=checkpoint.go:384 msg="flushing series" totalSize="169 kB" series=1
level=info ts=2026-05-04T23:54:15.299319886Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-04T23:54:15.300550038Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00871144s
ts=2026-05-04T23:54:15.347760732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38090"
level=info ts=2026-05-04T23:54:18.370686173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:54:18.375972207Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:54:20.29147493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:54:20.857094301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:54:20.861419739Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:54:20.861443219Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:54:20.861468147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:54:20.864859183Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:54:21.155724044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:54:21.158848466Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T23:54:23.451096508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:54:23.457590795Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T23:54:25.290724451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:54:25.503496522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=warn ts=2026-05-04T23:54:30.291118292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:54:30.339294574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:54:30.342811642Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-04T23:54:30.356880929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:54:30.362943192Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:54:30.362961268Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:54:30.362983763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:54:30.370039684Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:54:32.302201325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:54:32.308948259Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:54:32.308972378Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:54:32.308997089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:54:32.314209691Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T23:54:33.240695105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:54:33.244106311Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T23:54:35.291146047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:54:37.436204771Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:54:40.291449874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:54:40.495382206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:54:40.499638276Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:54:41.414039105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:54:41.422666436Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:54:42.247560837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:54:42.25081659Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T23:54:45.272695616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:54:45.272696858Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:54:45.291063358Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-04T23:54:45.291064586Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-04T23:54:45.291086371Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:54:45.291096236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:54:45.291104801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:54:45.291305097Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-04T23:54:45.29131945Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:54:45.29135511Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=28.889µs success=true
level=debug ts=2026-05-04T23:54:45.29157479Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-04T23:54:45.34937594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37452"
level=info ts=2026-05-04T23:54:45.977809991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:54:45.983603474Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:54:48.62517853Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=warn ts=2026-05-04T23:54:50.290583371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:54:51.959298467Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-04T23:54:54.495882697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:54:54.500817376Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-04T23:54:54.904988903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55780"
level=warn ts=2026-05-04T23:54:55.290698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:54:55.291777104Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-05-04T23:54:55.504430929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-04T23:54:55.60824934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:54:55.611934778Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T23:54:56.949089022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:54:56.956676102Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:54:57.345926807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:54:57.352003402Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-04T23:54:58.625967797Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-04T23:54:59.046055612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:54:59.049397089Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T23:55:00.291533578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:55:00.322670369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:55:00.325990189Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T23:55:01.439551778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:55:01.443852936Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:55:01.959291556Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-04T23:55:05.291340785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:05.292484744Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=info ts=2026-05-04T23:55:05.512890938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:55:05.516131501Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T23:55:05.8164325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:55:07.139939474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:55:07.14634873Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:55:08.104283413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:55:08.110582197Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T23:55:08.244812067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:55:08.248584128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:55:08.641585589Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T23:55:08.641607196Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=warn ts=2026-05-04T23:55:10.291170898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:11.967203385Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-04T23:55:11.967230625Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=info ts=2026-05-04T23:55:12.742725181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:55:12.746432151Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T23:55:12.790720973Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:55:14.622131424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:55:14.629322717Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:55:15.291306364Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:55:15.291338272Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:55:15.291348546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:55:15.291336282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:15.292460861Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-04T23:55:16.630947451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:55:16.637505082Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:55:18.626008113Z caller=checkpoint.go:372 msg="writing series" size="568 kB"
ts=2026-05-04T23:55:19.477903977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38958"
level=warn ts=2026-05-04T23:55:20.29155396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:21.958441861Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=info ts=2026-05-04T23:55:24.714247262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:55:24.718581818Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T23:55:24.896150519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:55:24.89985085Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T23:55:25.291236919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:25.292295241Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-04T23:55:25.506176863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:55:28.626742015Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-04T23:55:28.626761796Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=5
level=warn ts=2026-05-04T23:55:30.290823043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:31.959337175Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-04T23:55:35.29061225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:35.291715394Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=debug ts=2026-05-04T23:55:38.625810304Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T23:55:40.29151443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:55:41.481183096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:55:41.486326836Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T23:55:42.004954776Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-05-04T23:55:42.004993492Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=4
level=debug ts=2026-05-04T23:55:43.630542869Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:55:44.796814049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:55:44.800795999Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T23:55:45.272559207Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:55:45.273733582Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:55:45.290965441Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:55:45.290987234Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:55:45.290995195Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:55:45.291032194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:45.292284621Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-04T23:55:46.660390619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:55:46.663926704Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T23:55:48.128381375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:55:48.132051044Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T23:55:48.426860849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:55:48.430129469Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T23:55:48.625787181Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=info ts=2026-05-04T23:55:48.801793558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:55:48.805574957Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T23:55:49.034091308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:55:49.041907648Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:55:49.041929218Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:55:49.041958642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:55:49.050706633Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:55:49.728016823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:55:49.734464718Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-04T23:55:50.290959889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:51.959143266Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-04T23:55:52.829934605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:55:52.834045414Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T23:55:55.291500043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:55:55.291820328Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-04T23:55:55.291831582Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-04T23:55:55.507941841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:55:56.050495631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:55:56.058600078Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:55:56.679699838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:55:56.688771742Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:55:58.625590101Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-04T23:55:58.688111335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:55:58.691543544Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T23:56:00.291018437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:56:00.902163608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:56:00.905722043Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T23:56:00.996949154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:56:01.006796653Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:56:01.431713203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:56:01.488053536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:56:01.495149068Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:56:01.926243841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:56:01.930966972Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:56:01.959636304Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=warn ts=2026-05-04T23:56:05.291222964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:56:05.292794073Z caller=checkpoint.go:372 msg="writing series" size="587 kB"
level=debug ts=2026-05-04T23:56:05.292810957Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-04T23:56:05.60548781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:56:05.609044948Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-04T23:56:08.625534718Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-04T23:56:10.291576654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:56:10.350838192Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:56:10.564207131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:56:10.569286324Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T23:56:11.959344311Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=warn ts=2026-05-04T23:56:15.29088501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:56:15.290909126Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:56:15.290937523Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:56:15.290951793Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:56:15.292117953Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-04T23:56:18.625817804Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
ts=2026-05-04T23:56:19.48201691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59844"
level=warn ts=2026-05-04T23:56:20.291353234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:56:21.959559889Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T23:56:25.291290563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:56:25.292640911Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
ts=2026-05-04T23:56:25.510078836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:56:28.625213743Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-04T23:56:28.706627377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:56:28.711240783Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-04T23:56:30.291174847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:56:31.959428005Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-04T23:56:31.959451582Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=8
level=info ts=2026-05-04T23:56:32.432366721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:56:32.44001007Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:56:33.516284021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:56:33.520839519Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T23:56:33.690470155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:56:33.694077974Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T23:56:35.13115939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:56:35.134585917Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T23:56:35.291146246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:56:35.293035023Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T23:56:35.293050771Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-05-04T23:56:36.805312932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:56:36.808604827Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T23:56:38.625809227Z caller=checkpoint.go:372 msg="writing series" size="872 kB"
level=info ts=2026-05-04T23:56:39.796940895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:56:39.806108701Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T23:56:40.291275315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:56:41.021344003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:56:41.029281238Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:56:41.387477998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:56:41.390999329Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T23:56:41.803029896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:56:41.810045595Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:56:41.95855494Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-05-04T23:56:44.151455112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:56:44.155809299Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:56:44.344592799Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:56:45.272513978Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:56:45.273047098Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:56:45.291390633Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:56:45.291411786Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:56:45.291419193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:56:45.291416668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:56:45.292481758Z caller=checkpoint.go:372 msg="writing series" size="991 B"
level=debug ts=2026-05-04T23:56:47.555162544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:56:48.398728862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:56:48.405457496Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:56:48.625777834Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-04T23:56:48.669059206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:56:48.672522208Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T23:56:50.290862463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:56:51.958409054Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-04T23:56:51.958434504Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-04T23:56:53.476893333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:56:53.480199087Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T23:56:54.213107704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:56:54.216454458Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:56:54.216476714Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:56:54.216501399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:56:54.219557804Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T23:56:55.291473402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:56:55.295020074Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-04T23:56:55.295038299Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=1
ts=2026-05-04T23:56:55.511367467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:56:57.588640867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:56:57.59340087Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T23:56:57.761812593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:56:57.766363047Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T23:56:58.625609862Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-04T23:57:00.291474974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:57:01.958521476Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-04T23:57:05.291273648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:57:05.292467279Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-04T23:57:08.288549207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:57:08.294784235Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T23:57:08.626985146Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-04T23:57:08.627007529Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=4
level=warn ts=2026-05-04T23:57:10.290817514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:57:11.958881864Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-04T23:57:15.182785518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:57:15.187326422Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:57:15.274027707Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T23:57:15.291216134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:57:15.291220338Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:57:15.291246785Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:57:15.291257218Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:57:15.292400465Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
ts=2026-05-04T23:57:15.357583009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35018"
level=info ts=2026-05-04T23:57:17.34924958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:57:17.355357888Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:57:17.533013663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:57:17.537094777Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:57:17.537116428Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:57:17.537136673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:57:17.540113046Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T23:57:18.625241612Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T23:57:19.053394664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:57:19.0572179Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:57:19.057257977Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:57:19.057301587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:57:19.060295394Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-04T23:57:20.291685076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:57:21.958619164Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-04T23:57:22.543468461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:57:22.550445986Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T23:57:23.219239757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:57:23.222740434Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-04T23:57:23.222761898Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:57:23.222783669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:57:23.22607131Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T23:57:23.980978913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:57:23.984091793Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df52ee4a8%3A19df52f57bf%3Aa9d93aed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T23:57:23.984112753Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:57:23.984134262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="121 B" avg_comp="121 B" total_uncomp="218 kB" avg_uncomp="218 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-04T23:57:23.986851756Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-04T23:57:24.913155528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41800"
level=warn ts=2026-05-04T23:57:25.290739865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:57:25.291911812Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T23:57:25.51302645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:57:28.625782672Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-04T23:57:30.290781514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=warn ts=2026-05-04T23:57:30.290803088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-04T23:57:30.294244318Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T23:57:31.959209125Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-04T23:57:32.761964151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:57:32.763653855Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T23:57:32.780838123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:57:32.783683971Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:57:32.783701551Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:57:32.783722475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:57:32.786382133Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T23:57:33.098314409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:57:33.101541021Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T23:57:34.84879852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:57:34.851998113Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T23:57:35.290876151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:57:35.292605032Z caller=checkpoint.go:372 msg="writing series" size="616 kB"
level=debug ts=2026-05-04T23:57:35.292619341Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-05-04T23:57:35.843557721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:57:35.851795489Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:57:35.859877832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:57:35.863377675Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:57:38.625225235Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-04T23:57:39.757498032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:57:39.761002854Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T23:57:40.290990962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:57:40.506795122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-04T23:57:40.510218172Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-04T23:57:41.179367621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:57:41.186181365Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T23:57:41.37187436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:57:41.375179189Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-04T23:57:41.959003939Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-04T23:57:43.786455495Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:57:45.272550203Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:57:45.272763024Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:57:45.291118519Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:57:45.291157061Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:57:45.291166603Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:57:45.2911635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:57:45.292897524Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=info ts=2026-05-04T23:57:45.350536864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:57:45.353925018Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T23:57:46.121615544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:57:46.126746531Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:57:46.246997891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:57:46.254640291Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T23:57:47.337186994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:57:47.341046151Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:57:47.341071844Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:57:47.341103315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:57:47.343988698Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T23:57:48.625545938Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-04T23:57:49.23221409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:57:49.235921623Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T23:57:50.290647614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:57:51.95915073Z caller=checkpoint.go:372 msg="writing series" size="468 kB"
level=debug ts=2026-05-04T23:57:51.959239521Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-04T23:57:55.291348975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:57:55.29240888Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-04T23:57:55.514650559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:57:58.571605548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:57:58.575079412Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T23:57:58.625467394Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=warn ts=2026-05-04T23:58:00.291109575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:58:00.837109051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:58:00.841806702Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T23:58:01.769608278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:58:01.7737334Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:58:01.959381249Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-04T23:58:02.783768775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:58:02.787165655Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T23:58:03.182256108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-04T23:58:03.18653008Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-04T23:58:03.544666089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:58:03.548033856Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T23:58:05.290635901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:58:05.291916229Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=info ts=2026-05-04T23:58:06.721178193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:58:06.728147676Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T23:58:08.625412106Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-04T23:58:08.625447808Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-04T23:58:10.290804261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:58:10.905208611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:58:10.912666126Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-04T23:58:11.959454795Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-04T23:58:14.853129208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:58:15.130251118Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:58:15.291151425Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:58:15.291181303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:58:15.291190666Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:58:15.291191603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:58:15.2923293Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-04T23:58:15.292341747Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-05-04T23:58:18.625071049Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=warn ts=2026-05-04T23:58:20.291466576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:58:21.958407842Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T23:58:22.049714977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="804 B" avg_comp="804 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:58:22.05438439Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T23:58:23.838652986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:58:23.846566872Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T23:58:24.917061991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34436"
level=warn ts=2026-05-04T23:58:25.290762388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:58:25.291918534Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-04T23:58:25.516340701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:58:25.755195338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:58:25.758638977Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T23:58:26.308684798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:58:26.312208499Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:58:26.312227533Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:58:26.312248541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:58:26.314814802Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-04T23:58:26.321919326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:58:26.327126145Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T23:58:28.625858701Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=warn ts=2026-05-04T23:58:30.290592389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:58:30.791094879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:58:30.794558857Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:58:30.794581186Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:58:30.794602553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:58:30.797189474Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T23:58:31.960026948Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-04T23:58:31.960050278Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=5
level=info ts=2026-05-04T23:58:32.702178302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:58:32.707128205Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T23:58:34.839861082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:58:34.843557541Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:58:35.290849265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:58:35.29210523Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-04T23:58:38.625233364Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=warn ts=2026-05-04T23:58:40.291388538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:58:40.625469658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:58:40.631747654Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:58:41.959362089Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-04T23:58:42.503230016Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:58:43.824044256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:58:43.829019387Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-04T23:58:44.551880501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:58:44.557182759Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-04T23:58:44.557212141Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:58:44.557241681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:58:44.561781379Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:58:44.576021958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:58:44.578847242Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T23:58:45.27264146Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:58:45.273004919Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-05-04T23:58:45.291383564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:58:45.291395676Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:58:45.291412049Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:58:45.291420021Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T23:58:45.292496522Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-04T23:58:45.562235443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-04T23:58:45.565366582Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-04T23:58:48.625910285Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-04T23:58:49.756022902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:58:49.75939714Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T23:58:50.29092571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:58:51.958855052Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T23:58:51.958876931Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-04T23:58:55.070162501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:58:55.074458507Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T23:58:55.291096293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:58:55.292391717Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-05-04T23:58:55.518666372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-04T23:58:56.996668195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:58:56.999927183Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T23:58:58.625571044Z caller=checkpoint.go:372 msg="writing series" size="415 kB"
level=warn ts=2026-05-04T23:59:00.291607027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:59:01.307319404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:59:01.311606123Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:59:01.958305679Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-04T23:59:02.201081792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:59:02.207207053Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:59:03.86996297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:59:03.876852402Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T23:59:05.159416711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T23:59:05.16574869Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T23:59:05.291279565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:59:05.292352822Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-04T23:59:08.627822822Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-04T23:59:08.627854538Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=5
level=warn ts=2026-05-04T23:59:10.291256385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:59:11.592837552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T23:59:11.598195586Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T23:59:11.959357336Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=info ts=2026-05-04T23:59:12.777959229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:59:12.781603632Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T23:59:13.385998855Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:59:13.386064993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="198 B" avg_comp="198 B" total_uncomp="615 B" avg_uncomp="615 B" idle=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-04T23:59:13.389395966Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-04T23:59:15.291534575Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:59:15.291537082Z caller=checkpoint.go:384 msg="flushing series" totalSize="176 kB" series=1
level=debug ts=2026-05-04T23:59:15.291559584Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:59:15.291567385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:59:15.291582959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:59:15.299978057Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-04T23:59:15.301438474Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00991183s
level=debug ts=2026-05-04T23:59:15.437868626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T23:59:19.854404926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T23:59:19.857973879Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T23:59:20.291324465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:59:20.787667245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T23:59:20.790977935Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T23:59:21.321471545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T23:59:21.32840673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T23:59:23.894303809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T23:59:23.89989841Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T23:59:24.21672457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-04T23:59:24.2201771Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=warn ts=2026-05-04T23:59:25.290717042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T23:59:25.520203986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-04T23:59:25.630073737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T23:59:25.634184636Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T23:59:28.301861256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-04T23:59:28.305027654Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-04T23:59:28.749064943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T23:59:28.753872536Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T23:59:30.29115196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:59:33.378910235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:59:33.383035547Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:59:33.383057498Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:59:33.383099604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:59:33.386228376Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T23:59:35.290653981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:59:38.141797911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T23:59:38.14829016Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T23:59:38.446630293Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T23:59:39.817945389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T23:59:39.82139473Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-04T23:59:40.290654927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:59:40.787208915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T23:59:40.790844084Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T23:59:42.302134264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T23:59:42.306082478Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T23:59:43.835405714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:59:43.84281744Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:59:43.842847682Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T23:59:43.842879363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-04T23:59:43.848845739Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-04T23:59:45.272698444Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T23:59:45.273057068Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-04T23:59:45.290419618Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T23:59:45.290565183Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-04T23:59:45.290605185Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T23:59:45.290600842Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T23:59:45.290623408Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T23:59:45.290634382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T23:59:45.290632159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:59:45.290642851Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=28.268µs success=true
level=debug ts=2026-05-04T23:59:45.290867585Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-04T23:59:45.367798694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48842"
level=info ts=2026-05-04T23:59:46.241119052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T23:59:46.246743241Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T23:59:48.62494379Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
ts=2026-05-04T23:59:49.497030348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:55458"
level=warn ts=2026-05-04T23:59:50.291208343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T23:59:51.446031367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T23:59:51.45009582Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T23:59:51.957792504Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-04T23:59:53.213316468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-04T23:59:53.217336531Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=warn ts=2026-05-04T23:59:55.290629404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T23:59:55.291720518Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-04T23:59:55.483021839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T23:59:55.486233943Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-04T23:59:55.5215559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:59:58.625378484Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-05T00:00:00.290597198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:00:00.744415614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:00:00.753169541Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:00:01.50590059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:00:01.513173139Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:00:01.957847873Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T00:00:02.285084276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:00:02.289986113Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:00:04.911333015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:00:04.915639932Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:00:05.291617614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:00:05.291709826Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-05T00:00:05.344808451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:00:05.348505274Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:00:05.881062865Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:00:08.388938763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:00:08.44897843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:00:08.457069658Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:00:08.641492721Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-05T00:00:08.641518022Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=warn ts=2026-05-05T00:00:10.291455984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:00:11.338911065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:00:11.342732848Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:00:11.963097961Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-05T00:00:11.963120817Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=warn ts=2026-05-05T00:00:15.290925865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:00:15.290945103Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-05T00:00:15.29093915Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:00:15.290968499Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:00:15.290981259Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T00:00:16.707203429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-05T00:00:16.714100234Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-05T00:00:18.625137785Z caller=checkpoint.go:372 msg="writing series" size="590 kB"
level=warn ts=2026-05-05T00:00:20.291398442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:00:21.270774463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:00:21.277005864Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T00:00:21.957786502Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-05-05T00:00:22.406301829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:00:22.413110827Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:00:22.413138813Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:00:22.41316746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:00:22.418804383Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:00:22.921179003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:00:22.924760985Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:00:24.897653492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:00:24.901676376Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-05T00:00:25.290659679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:00:25.2916659Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-05T00:00:25.52346526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:00:27.824838953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:00:27.828475547Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:00:28.62603724Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T00:00:28.626064938Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=5
level=warn ts=2026-05-05T00:00:30.291661803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:00:31.957960762Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-05T00:00:32.514750416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:00:32.518562547Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:00:32.989872583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:00:32.996746878Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:00:35.11579271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:00:35.119450064Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T00:00:35.291396394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:00:35.291521022Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-05T00:00:35.303536716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:00:35.306919561Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:00:37.017044115Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:00:37.612863809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:00:37.619409235Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:00:38.624435864Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-05T00:00:40.291512051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:00:42.003976453Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-05T00:00:42.004003707Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=4
level=info ts=2026-05-05T00:00:42.076068288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:00:42.080735974Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:00:42.080761347Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:00:42.08078967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:00:42.084935623Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:00:43.602321202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:00:43.606392377Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:00:45.272561831Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:00:45.272561772Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-05T00:00:45.273833105Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-05T00:00:45.273871435Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975487 n=1
level=debug ts=2026-05-05T00:00:45.273910769Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975487 n=1
level=info ts=2026-05-05T00:00:45.290928682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T00:00:45.290938745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:00:45.290957922Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:00:45.29096622Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:00:45.290995909Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=info ts=2026-05-05T00:00:46.455669201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:00:46.463515797Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:00:46.463542953Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:00:46.463567732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:00:46.470715545Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:00:46.774547163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:00:46.779618739Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:00:47.644179728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:00:47.649651742Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:00:48.625712695Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
ts=2026-05-05T00:00:49.501942995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48164"
level=warn ts=2026-05-05T00:00:50.291087483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:00:50.957406928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:00:50.9611454Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:00:51.859224505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:00:51.958684288Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-05T00:00:54.150835369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:00:54.159043057Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-05T00:00:54.925801199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44926"
level=warn ts=2026-05-05T00:00:55.290849329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:00:55.291190284Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-05T00:00:55.291201851Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-05T00:00:55.524885561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:00:58.625194634Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-05T00:00:59.014384896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:00:59.018578081Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T00:01:00.291387321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:01.95991219Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-05T00:01:03.499467713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:01:03.507981146Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T00:01:05.291281172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:05.291578934Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
level=info ts=2026-05-05T00:01:05.352476529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:01:05.360006535Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:01:07.470979744Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:01:07.471032159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:01:07.474887504Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T00:01:08.625464197Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-05T00:01:08.625486933Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-05T00:01:08.8489434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:01:08.853418112Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-05T00:01:10.291233283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:01:10.379571755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:01:10.382978884Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:01:11.95843815Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-05T00:01:15.12482416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:01:15.134304335Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:01:15.134346389Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:01:15.134383182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:01:15.141419443Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T00:01:15.290968684Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:01:15.290995486Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:01:15.291005512Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:01:15.291015368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:15.291071398Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T00:01:16.087954937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:01:16.091440259Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T00:01:16.247089552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-05T00:01:16.252428102Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-05T00:01:17.129151415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:01:17.135567231Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T00:01:18.625346768Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-05T00:01:20.291356099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:21.958544214Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-05T00:01:23.3334629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:01:23.337538418Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:01:24.072622476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:01:24.079223894Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
ts=2026-05-05T00:01:24.927009803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53898"
level=warn ts=2026-05-05T00:01:25.290982236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:25.291103482Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
ts=2026-05-05T00:01:25.526857433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:01:26.192940273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:01:26.197228311Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:01:27.595626248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:01:27.599563132Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:01:28.177452477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:01:28.180870732Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:01:28.625052575Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-05T00:01:28.811581571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:01:28.815487856Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T00:01:30.29150909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:31.95907062Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-05T00:01:31.959095651Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=debug ts=2026-05-05T00:01:34.116839614Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:01:35.029397287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:01:35.036205708Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T00:01:35.291417841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:35.292151647Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-05T00:01:35.292170016Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-05T00:01:35.46984151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:01:35.474093642Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:01:38.579554725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:01:38.583042435Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:01:38.625572275Z caller=checkpoint.go:372 msg="writing series" size="926 kB"
level=warn ts=2026-05-05T00:01:40.291711722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:41.958050116Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=debug ts=2026-05-05T00:01:41.958074539Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-05T00:01:43.2015958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:01:43.206048463Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:01:45.18595259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:01:45.193974644Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:01:45.27246947Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:01:45.273516854Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T00:01:45.290730527Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:01:45.290759087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:01:45.290767676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:01:45.290832648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:45.29088364Z caller=checkpoint.go:372 msg="writing series" size="954 B"
level=debug ts=2026-05-05T00:01:45.681186654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:01:45.683033687Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T00:01:47.043941809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:01:47.047631499Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:01:48.62489515Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T00:01:49.14941241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:01:49.157131881Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T00:01:50.291486609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:51.958508802Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T00:01:53.133177306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:01:53.13691866Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T00:01:55.291369243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:01:55.29523877Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-05T00:01:55.295255301Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=4
ts=2026-05-05T00:01:55.528528799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:01:55.800792667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:01:55.806762935Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T00:01:56.039914247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:01:56.04886299Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:01:58.62480468Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-05T00:01:58.960224535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:01:58.965783829Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:02:00.291620196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:02:01.958255206Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-05T00:02:03.911191004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:02:03.91494577Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:02:04.489673053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:02:04.493819481Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T00:02:05.291425296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:02:05.291606664Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=debug ts=2026-05-05T00:02:06.61665904Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:02:07.496279578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-05T00:02:07.501712356Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-05T00:02:08.626753487Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T00:02:08.626775287Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=4
level=warn ts=2026-05-05T00:02:10.290732604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:02:11.958463576Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-05T00:02:13.270157162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:02:13.273712603Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T00:02:14.959294057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:02:14.96594285Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:02:15.290786743Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:02:15.290810952Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T00:02:15.290798019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:02:15.29081887Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:02:15.290915589Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
ts=2026-05-05T00:02:15.376346063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47324"
level=info ts=2026-05-05T00:02:16.273972843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:02:16.27716423Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:02:18.624532945Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-05T00:02:20.291307044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:02:21.958076989Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=debug ts=2026-05-05T00:02:22.017132827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:02:22.23468054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:02:22.237864771Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:02:22.942981513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:02:22.946577582Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:02:24.164219552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="630 B" avg_comp="630 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:02:24.167484827Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
ts=2026-05-05T00:02:24.930997374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54790"
level=warn ts=2026-05-05T00:02:25.290691693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:02:25.291825863Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-05T00:02:25.53035439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:02:28.309945261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:02:28.314223564Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:02:28.624885778Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-05T00:02:30.290882108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:02:30.440230465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:02:30.447100296Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:02:31.958373021Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-05T00:02:32.853058113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:02:32.858517681Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-05T00:02:33.88950087Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:02:34.129119396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:02:34.132520729Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T00:02:35.290684796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:02:35.292282682Z caller=checkpoint.go:372 msg="writing series" size="509 kB"
level=debug ts=2026-05-05T00:02:35.292300513Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=debug ts=2026-05-05T00:02:38.624342669Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-05T00:02:39.806672167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:02:39.813570686Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T00:02:40.29154977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:02:41.958144576Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-05T00:02:44.268685533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:02:44.272279183Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:02:45.272542783Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:02:45.272839584Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:02:45.291178539Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:02:45.291197694Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:02:45.291206069Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:02:45.291284519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:02:45.291515271Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=info ts=2026-05-05T00:02:45.936382796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:02:45.939906148Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:02:47.114165811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:02:47.118033744Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:02:48.200011605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:02:48.20359939Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T00:02:48.625102752Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-05T00:02:49.371398513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:02:49.374354181Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-05T00:02:49.511514399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34798"
level=warn ts=2026-05-05T00:02:50.291059916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:02:51.902022948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:02:51.91063262Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:02:51.910655932Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:02:51.910686212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:02:51.917465975Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:02:51.959036286Z caller=checkpoint.go:372 msg="writing series" size="539 kB"
level=debug ts=2026-05-05T00:02:51.959059648Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-05T00:02:52.223397984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:02:52.229448859Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T00:02:55.290791131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:02:55.291932757Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-05T00:02:55.531628471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:02:58.625368761Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=info ts=2026-05-05T00:02:59.052278104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:02:59.055741548Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T00:03:00.290648512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:03:01.958486495Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=debug ts=2026-05-05T00:03:03.483015646Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:03:04.650383688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-05T00:03:04.655677397Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-05T00:03:05.291092286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:03:05.291313376Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=debug ts=2026-05-05T00:03:05.291327815Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-05T00:03:07.693994309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:03:07.701429405Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:03:08.624284357Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-05T00:03:09.45170316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:03:09.455213888Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:03:09.663847725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:03:09.6671718Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:03:10.032115418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:03:10.036549634Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:03:10.054589532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:03:10.06021091Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-05T00:03:10.290742739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:03:11.958253564Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T00:03:11.958299064Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-05T00:03:13.352696411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:03:13.359519143Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:03:15.290731021Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:03:15.290769108Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:03:15.290779575Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:03:15.290815573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:03:15.290926514Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-05T00:03:16.396432525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:03:16.408463751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:03:16.411406298Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:03:16.63203848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:03:16.635333588Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:03:18.624961192Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=warn ts=2026-05-05T00:03:20.29164121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:03:21.95828356Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-05T00:03:23.715316851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:03:23.720547007Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:03:24.720447548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="804 B" avg_comp="804 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:03:24.724003138Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T00:03:25.291096274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:03:25.291167606Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-05T00:03:25.532593969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:03:26.239257627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:03:26.242647036Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:03:26.24266768Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:03:26.242692022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:03:26.24578953Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:03:27.467761412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:03:27.474198215Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T00:03:28.624777658Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=warn ts=2026-05-05T00:03:30.290571912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:03:31.650729526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:03:31.660235137Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:03:31.960137581Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-05T00:03:31.960159231Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=6
level=debug ts=2026-05-05T00:03:33.160900801Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:03:33.629235988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:03:33.63477577Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T00:03:35.291508727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:03:35.291644406Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-05T00:03:38.625090249Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-05T00:03:38.72676889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:03:38.731581748Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T00:03:40.29113986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:03:41.711476602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:03:41.715457318Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:03:41.794711685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:03:41.79858778Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-05T00:03:41.958129719Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-05T00:03:44.169072131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:03:44.172767241Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:03:44.172787879Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:03:44.172809168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:03:44.175368054Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:03:45.272136972Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:03:45.273196522Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-05T00:03:45.291588899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:03:45.291600436Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:03:45.291612232Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-05T00:03:45.291622449Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:03:45.2916324Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T00:03:45.380320085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:50130"
level=info ts=2026-05-05T00:03:45.561225685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:03:45.564436952Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:03:48.625003468Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-05T00:03:50.29104123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:03:51.958386666Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-05T00:03:51.95840929Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
ts=2026-05-05T00:03:54.936335943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33146"
level=warn ts=2026-05-05T00:03:55.291560612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:03:55.291752555Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
ts=2026-05-05T00:03:55.533713543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:03:56.896965631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:03:56.905806681Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:03:57.408483664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:03:57.412344675Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-05T00:03:57.412364753Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:03:57.412385379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:03:57.415256013Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T00:03:57.637894364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:03:57.641605084Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:03:58.624772594Z caller=checkpoint.go:372 msg="writing series" size="572 kB"
level=warn ts=2026-05-05T00:04:00.291003823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:04:01.95831835Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-05T00:04:02.026616725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:04:02.030342537Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:04:03.067843434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:04:03.452917711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-05T00:04:03.458327837Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-05T00:04:04.256584102Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:04:05.063933153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:04:05.067379526Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:04:05.067404528Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:04:05.067426587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:04:05.070736332Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T00:04:05.291199355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:04:05.291230776Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-05T00:04:06.226606876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:04:06.230069794Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:04:06.23009034Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:04:06.230112052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:04:06.233016835Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:04:06.603382063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:04:06.610205653Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:04:07.651116548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:04:07.658077333Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:04:08.192630468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:04:08.199671602Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:04:08.535648955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:04:08.53920114Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:04:08.62647233Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-05T00:04:08.626493166Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=5
level=warn ts=2026-05-05T00:04:10.291177531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:04:11.958388771Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-05T00:04:14.165836683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:04:14.169985925Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:04:14.170005264Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:04:14.170027335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:04:14.172638118Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:04:15.291370797Z caller=checkpoint.go:384 msg="flushing series" totalSize="190 kB" series=1
level=info ts=2026-05-05T00:04:15.291421094Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T00:04:15.29142278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:04:15.291440797Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:04:15.29144843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T00:04:15.296805138Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-05T00:04:15.29812541Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007304499s
ts=2026-05-05T00:04:15.381318898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48710"
level=info ts=2026-05-05T00:04:15.608409397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:04:15.611946163Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T00:04:17.022988657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:04:17.030027402Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:04:17.51858833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:04:17.521992977Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-05T00:04:20.291194577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:04:22.513000815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:04:22.516338972Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:04:22.693005541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:04:22.696246605Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:04:24.347699559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:04:24.350898389Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T00:04:25.290828154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:04:25.456573078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:04:25.462944234Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T00:04:25.535326471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:04:27.049018568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:04:27.05364352Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:04:29.88816054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:04:29.891709435Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T00:04:30.290886034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:04:31.544388497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:04:31.547949478Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:04:31.547969647Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:04:31.54799086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:04:31.551100103Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:04:33.140922604Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:04:35.143622922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:04:35.147172917Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:04:35.291542948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:04:37.621924542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:04:37.626008033Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:04:40.291338384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:04:45.272521043Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:04:45.272843604Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:04:45.291174013Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:04:45.291192308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:04:45.291200224Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:04:45.291203962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:04:45.291217876Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T00:04:45.291350854Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-05T00:04:45.291359122Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T00:04:45.291377708Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.606µs success=true
level=debug ts=2026-05-05T00:04:45.291546408Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-05T00:04:45.382582121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51448"
level=info ts=2026-05-05T00:04:47.263497615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:04:47.26684634Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T00:04:47.625035647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:04:47.628663663Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:04:47.643806186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:04:47.646875154Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:04:48.488446035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-05T00:04:48.493207957Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-05T00:04:48.625258062Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-05T00:04:48.800985309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:04:48.804457665Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T00:04:50.291426792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:04:51.956993867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:04:51.958315166Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=error ts=2026-05-05T00:04:51.960700156Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:04:52.805533858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:04:52.811964626Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:04:53.232095956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:04:53.235361536Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:04:53.402623736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:04:53.408490386Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:04:53.801560945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:04:53.804701361Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:04:53.804723414Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:04:53.804744343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:04:53.807308775Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:04:53.96570445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:04:53.97153396Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T00:04:54.40500366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T00:04:55.29162796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:04:55.291653095Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-05-05T00:04:55.537134386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:04:58.625507758Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-05-05T00:04:59.673828318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T00:05:00.291248596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:05:01.390032201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:05:01.393458665Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:05:01.959004459Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-05T00:05:05.291607617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:05:05.291733791Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-05T00:05:07.775056128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:05:07.782807147Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:05:08.633528607Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-05T00:05:08.633550859Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=info ts=2026-05-05T00:05:08.776633755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:05:08.780357161Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:05:08.780382803Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:05:08.780405288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:05:08.783406106Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:05:09.240444334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:05:09.244110675Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T00:05:10.291154677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:05:11.966989359Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T00:05:11.967011999Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=1
level=info ts=2026-05-05T00:05:12.117331306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:05:12.12282922Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T00:05:15.291396588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:05:15.291398506Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:05:15.291427567Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:05:15.291437704Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:05:15.292491195Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-05T00:05:17.034402789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:05:17.038075182Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:05:17.400450903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:05:17.403607274Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-05T00:05:18.625875741Z caller=checkpoint.go:372 msg="writing series" size="434 kB"
level=info ts=2026-05-05T00:05:20.221736925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:05:20.225637839Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T00:05:20.290873223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:05:21.958880856Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
ts=2026-05-05T00:05:24.940569936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49736"
level=warn ts=2026-05-05T00:05:25.290923433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:05:25.29201501Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-05T00:05:25.538745457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:05:27.010099508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:05:27.014089357Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:05:27.098469598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:05:27.101947361Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:05:28.627418625Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-05T00:05:28.627446986Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=5
level=info ts=2026-05-05T00:05:29.148991372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:05:29.153116804Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:05:30.29137241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:05:31.959281416Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-05T00:05:34.04412966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:05:34.050198798Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:05:34.347404906Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:05:34.347479477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:05:34.351769823Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T00:05:35.291193009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:05:35.292458399Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-05T00:05:35.657223352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:05:35.660435983Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T00:05:38.496761212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:05:38.500071636Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:05:38.541118306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:05:38.54414414Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:05:38.625876994Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-05T00:05:39.384557489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:05:39.391867075Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T00:05:40.291258926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:05:41.077806943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:05:41.084035879Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:05:41.800089232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:05:41.803541311Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:05:42.011890322Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-05T00:05:42.011917279Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=4
level=info ts=2026-05-05T00:05:43.373055187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:05:43.376389965Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:05:43.376416376Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:05:43.376440963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:05:43.379166477Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:05:43.5626272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="910 kB" avg_comp="910 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-05T00:05:43.56739574Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df52f575e%3A19df555d4a1%3A17c65f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-05T00:05:43.567420348Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:05:43.676865578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="740 kB" avg_comp="740 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:05:43.681365725Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:05:45.272554824Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:05:45.272749107Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:05:45.291112047Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:05:45.291148952Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:05:45.291159893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:05:45.291208306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:05:45.292372949Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-05T00:05:46.713284174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:05:46.71997534Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:05:48.627072468Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=warn ts=2026-05-05T00:05:50.291336638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:05:50.522583926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:05:50.527484008Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:05:51.61466335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:05:51.618771743Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:05:51.958796541Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-05-05T00:05:52.044232485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:05:52.045753601Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T00:05:53.638690342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:05:53.647850841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T00:05:55.290721007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:05:55.29216024Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-05T00:05:55.292173886Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-05T00:05:55.540248627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:05:57.785841835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:05:57.789470619Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:05:57.894812167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:05:57.898135839Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:05:58.394651994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:05:58.399381005Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T00:05:58.625934353Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-05T00:06:00.290621211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:06:01.959011746Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-05T00:06:02.438229038Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T00:06:05.291628786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:06:05.292130987Z caller=checkpoint.go:372 msg="writing series" size="600 kB"
level=debug ts=2026-05-05T00:06:05.292146182Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-05T00:06:07.762989994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:06:07.770771192Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T00:06:08.625869852Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-05T00:06:08.943598296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:06:08.947232235Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:06:10.08978082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:06:10.093137062Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T00:06:10.291212002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:06:11.607057127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:06:11.611575683Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:06:11.958635585Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-05T00:06:15.291439641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:06:15.291443349Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:06:15.291472632Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:06:15.291484062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:06:15.292570774Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T00:06:17.315408091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:06:17.319941144Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:06:17.319968199Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:06:17.320012513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:06:17.323450232Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:06:17.592350881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:06:17.60092953Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:06:18.625919085Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=info ts=2026-05-05T00:06:19.422959839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:06:19.42672444Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T00:06:20.29110352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:06:21.959625224Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-05T00:06:23.547387644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:06:23.55088106Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:06:23.627055616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:06:23.634537446Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:06:23.634560799Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:06:23.634588168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:06:23.640651781Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-05T00:06:24.942910035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:57132"
level=warn ts=2026-05-05T00:06:25.290664167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:06:25.291998343Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
ts=2026-05-05T00:06:25.542104202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:06:27.398534053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:06:27.40320289Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:06:27.911748502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:06:27.918436544Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:06:27.91845876Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:06:27.918485804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:06:27.924577062Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:06:28.13515647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:06:28.138574058Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:06:28.361398778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:06:28.365412485Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:06:28.625521191Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-05T00:06:28.873719378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T00:06:30.290645105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:06:30.672560283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:06:30.67582434Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:06:31.959683072Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-05T00:06:31.9597081Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=8
level=info ts=2026-05-05T00:06:34.37740261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:06:34.381003679Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T00:06:34.734050571Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T00:06:35.290885802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:06:35.29282477Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-05T00:06:35.292842044Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-05T00:06:35.61625544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:06:35.620029949Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:06:36.202342072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:06:36.206076972Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:06:36.867064963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:06:36.873693465Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:06:37.996237268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:06:38.004131049Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:06:38.518385871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:06:38.526730237Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:06:38.62633018Z caller=checkpoint.go:372 msg="writing series" size="798 kB"
level=info ts=2026-05-05T00:06:39.994991906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:06:40.001660759Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-05T00:06:40.291012825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:06:41.958929466Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-05T00:06:45.040002853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:06:45.045895951Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:06:45.272635249Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:06:45.272637738Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:06:45.29099145Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:06:45.291027821Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:06:45.291036343Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:06:45.291079909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:06:45.292103651Z caller=checkpoint.go:372 msg="writing series" size="954 B"
level=debug ts=2026-05-05T00:06:48.62601717Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T00:06:49.247916773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:06:49.252995429Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-05T00:06:49.526883473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45888"
level=warn ts=2026-05-05T00:06:50.291188231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:06:51.958948155Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T00:06:52.490820665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:06:52.494847999Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:06:54.449319387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:06:54.455476269Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T00:06:54.94467963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45282"
level=warn ts=2026-05-05T00:06:55.291524152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:06:55.294041265Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-05-05T00:06:55.294058859Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.1 MB" series=6
ts=2026-05-05T00:06:55.544249692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-05T00:06:58.625030103Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-05T00:07:00.270208925Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T00:07:00.291446655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:07:00.641963206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:07:00.64597461Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:07:00.734290625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:07:00.740581343Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:07:01.958394261Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-05T00:07:03.731641596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:07:03.738280431Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:07:05.031003312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:07:05.034677845Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T00:07:05.290602023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:07:05.291874168Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-05T00:07:05.617784235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:07:05.621230826Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:07:08.627792058Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T00:07:08.627821977Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=4
level=info ts=2026-05-05T00:07:09.007921036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:07:09.018702592Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:07:09.488459578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:07:09.492786186Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T00:07:10.290590946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:07:11.050205502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:07:11.05396261Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:07:11.959020445Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-05T00:07:13.421518482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:07:13.425342146Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:07:15.291201962Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T00:07:15.291210896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:07:15.291235135Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:07:15.291247521Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:07:15.292369588Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-05-05T00:07:17.919249452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:07:18.625985543Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
ts=2026-05-05T00:07:19.529181969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45198"
level=info ts=2026-05-05T00:07:20.23630873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:07:20.24382025Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:07:20.243847876Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:07:20.243877767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:07:20.25102243Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:07:20.291405264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:07:21.95930963Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=info ts=2026-05-05T00:07:22.052323265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:07:22.059749296Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T00:07:22.77721451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:07:22.781109355Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:07:24.98072219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="630 B" avg_comp="630 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:07:24.984127839Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=warn ts=2026-05-05T00:07:25.29144227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:07:25.292540808Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-05T00:07:25.546308126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:07:26.948958644Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:07:28.62521128Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-05T00:07:29.925542308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:07:29.929197109Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T00:07:30.290921812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:07:31.958601503Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-05T00:07:33.676103508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:07:33.680419678Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T00:07:35.291472311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:07:35.29183643Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-05T00:07:35.291849745Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=8
level=info ts=2026-05-05T00:07:37.467864794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:07:37.476453645Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:07:38.625730508Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-05T00:07:38.742932169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:07:38.749748482Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:07:40.291324217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:07:40.387330517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:07:40.390697631Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:07:41.954689425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:07:41.95858116Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=error ts=2026-05-05T00:07:41.95975121Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:07:41.959770306Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:07:41.959797157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:07:41.963396652Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:07:43.407064277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:07:43.414295776Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:07:43.41432849Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:07:43.414361876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:07:43.420448471Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T00:07:45.272523229Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:07:45.272813948Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:07:45.291174472Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:07:45.291206917Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:07:45.291215706Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:07:45.291257305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:07:45.292656084Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
ts=2026-05-05T00:07:45.39356916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49682"
level=info ts=2026-05-05T00:07:47.061322151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:07:47.064969238Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:07:47.494464171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:07:47.498102651Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:07:48.625095093Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
ts=2026-05-05T00:07:49.531677634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57326"
level=info ts=2026-05-05T00:07:49.985110387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:07:49.991385076Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T00:07:50.291109696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:07:51.596910496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:07:51.602064652Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:07:51.95948122Z caller=checkpoint.go:372 msg="writing series" size="471 kB"
level=debug ts=2026-05-05T00:07:51.959509987Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-05-05T00:07:53.838012077Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:07:54.188973547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T00:07:55.290597911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:07:55.291654965Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-05T00:07:55.548252673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:07:55.914092657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:07:55.919839683Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T00:07:56.713240241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:07:56.717009598Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:07:57.373126148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:07:57.378092247Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:07:57.438248431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:07:57.441403778Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:07:58.625482906Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=info ts=2026-05-05T00:07:59.211941621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:07:59.218489227Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-05T00:08:00.291306972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:08:01.959051325Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-05T00:08:02.485355012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:08:02.490020814Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:08:02.490057024Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:08:02.49009606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:08:02.4935006Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:08:03.713084121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:08:03.71713019Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T00:08:05.291062226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:08:05.292160463Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-05T00:08:05.619051219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:08:05.622553525Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:08:06.665582879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:08:06.668917952Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T00:08:08.625656504Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-05T00:08:08.692179517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:08:08.696128111Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:08:10.290834845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:08:10.6602495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:08:10.668687677Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:08:11.960233935Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-05T00:08:11.960257239Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=6
level=info ts=2026-05-05T00:08:12.666476172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:08:12.669805656Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:08:15.290944014Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:08:15.290982515Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:08:15.290998877Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:08:15.290993428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:08:15.29211744Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T00:08:16.882004148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:08:16.886937432Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:08:18.091184741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:08:18.099596291Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:08:18.62504209Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=info ts=2026-05-05T00:08:18.977229661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:08:18.980877665Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-05T00:08:19.533519516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51702"
level=info ts=2026-05-05T00:08:19.609146196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:08:19.612655252Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=warn ts=2026-05-05T00:08:20.291015852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:08:21.958584534Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-05T00:08:24.47443864Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T00:08:25.290817599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:08:25.291944269Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-05T00:08:25.550121832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:08:27.475244966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:08:27.481444764Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T00:08:28.03095608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:08:28.034556508Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:08:28.626031589Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-05T00:08:30.291493114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:08:31.502678729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="804 B" avg_comp="804 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:08:31.506743041Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:08:31.962745242Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-05T00:08:31.962786511Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=6
level=info ts=2026-05-05T00:08:32.340541318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:08:32.347064766Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:08:33.277853891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:08:33.281550778Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T00:08:35.291333733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:08:35.292536061Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-05T00:08:35.891761631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:08:35.895222882Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-05T00:08:38.625420287Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=warn ts=2026-05-05T00:08:40.290921789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:08:41.544934466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:08:41.549120829Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T00:08:41.958382508Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-05T00:08:42.030470458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:08:43.5673142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:08:43.573186217Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:08:44.958799641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:08:44.961818016Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:08:45.27270988Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:08:45.273070073Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:08:45.293335446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:08:45.293354642Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:08:45.293363651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:08:45.293365855Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-05T00:08:45.293395846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:08:46.519377597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:08:46.522772697Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:08:48.625002028Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
ts=2026-05-05T00:08:49.535730396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56074"
level=info ts=2026-05-05T00:08:49.968295483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:08:49.97153459Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:08:50.106943934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:08:50.110196287Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T00:08:50.291538304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:08:50.514521462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:08:50.517593888Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:08:50.848389769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:08:50.851833675Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:08:51.959405983Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-05T00:08:51.959431187Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=debug ts=2026-05-05T00:08:52.304571798Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:08:55.110411239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:08:55.116204991Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:08:55.291453863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:08:55.29276643Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
ts=2026-05-05T00:08:55.551719827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-05T00:08:58.625306452Z caller=checkpoint.go:372 msg="writing series" size="475 kB"
level=warn ts=2026-05-05T00:09:00.291505646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:09:01.958827102Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-05T00:09:05.290636354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:09:05.291768448Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-05T00:09:06.16806881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:09:06.175010357Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:09:08.626318558Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-05T00:09:08.626342222Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=5
level=info ts=2026-05-05T00:09:09.688782169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:09:09.696889568Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:09:10.291390984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:09:11.95873986Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-05T00:09:12.500453093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:09:12.50553269Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:09:12.783315865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:09:12.787166835Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:09:12.958655708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:09:12.96653508Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:09:12.966565396Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:09:12.966600132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:09:12.97432078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:09:13.600128156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:09:13.603792862Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:09:15.290693446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:09:15.290747815Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:09:15.290759696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:09:15.290752075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:09:15.291809075Z caller=checkpoint.go:384 msg="flushing series" totalSize="199 kB" series=1
level=info ts=2026-05-05T00:09:15.301467462Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-05T00:09:15.302945223Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011438322s
level=info ts=2026-05-05T00:09:16.497804538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:09:16.501143955Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T00:09:20.29152478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:09:20.795664166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:09:20.802140112Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:09:22.578941027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:09:22.583435897Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:09:23.738079602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:09:23.741402369Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:09:24.330580745Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:09:24.342606258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:09:24.34628214Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:09:25.132487448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:09:25.138431433Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T00:09:25.291613659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:09:25.554354532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:09:28.379125636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:09:28.386590055Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T00:09:30.291103351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:09:33.08171515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:09:33.085396058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:09:34.041729941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:09:34.048682581Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:09:35.191669059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:09:35.195515082Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:09:35.290733906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:09:35.498766241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:09:35.500373061Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T00:09:39.401453517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:09:39.405020013Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:09:40.291478761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:09:40.746613345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:09:40.749979722Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:09:44.241399453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:09:44.246198087Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:09:45.272549482Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:09:45.273051848Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:09:45.29044684Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T00:09:45.29054757Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:09:45.290576835Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:09:45.29058801Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:09:45.290663813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:09:45.290659033Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-05T00:09:45.290684873Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T00:09:45.290704063Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.466µs success=true
level=debug ts=2026-05-05T00:09:45.290900509Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-05T00:09:46.487707487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:09:46.4960773Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:09:46.815983849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:09:46.819839433Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T00:09:48.625160982Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=info ts=2026-05-05T00:09:49.298698474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:09:49.307915968Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T00:09:50.290755201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:09:51.958358107Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=debug ts=2026-05-05T00:09:54.848539768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:09:55.291574676Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T00:09:55.291601037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:09:55.556243576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:09:57.293959277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:09:57.299193495Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:09:58.625475445Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=warn ts=2026-05-05T00:10:00.290753181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:10:01.958471931Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T00:10:02.25032196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:10:02.25459036Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T00:10:05.291178243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:10:05.291392371Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=info ts=2026-05-05T00:10:07.299075765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:10:07.302953907Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:10:07.748277603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:10:07.752839162Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:10:08.308771798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:10:08.312494876Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:10:08.636651696Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-05T00:10:08.636681668Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=info ts=2026-05-05T00:10:09.032950331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:10:09.03661435Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:10:09.036638631Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:10:09.036661493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:10:09.039540146Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:10:09.489276561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:10:09.496291793Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:10:10.291346257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:10:11.970795273Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-05T00:10:11.970822792Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-05-05T00:10:15.290532398Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:10:15.290565366Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:10:15.290574586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:10:15.290664483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:10:15.291629976Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-05-05T00:10:15.401988484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49538"
level=info ts=2026-05-05T00:10:16.400571815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:10:16.404075405Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:10:17.605808646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:10:17.612945303Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T00:10:18.624705285Z caller=checkpoint.go:372 msg="writing series" size="458 kB"
level=info ts=2026-05-05T00:10:19.113810322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:10:19.118172264Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:10:19.443132197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:10:19.447075141Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:10:19.499452382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:10:19.5033341Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:10:19.503363938Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:10:19.503397457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:10:19.506645241Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-05T00:10:19.541112938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59028"
level=warn ts=2026-05-05T00:10:20.290936075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:10:21.957847186Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-05T00:10:23.204611067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:10:23.211036166Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:10:23.240051617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:10:23.246427996Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T00:10:25.291522929Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-05T00:10:25.291533118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:10:25.49831539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:10:25.502399807Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-05T00:10:25.557803247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:10:26.271599443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:10:27.486152908Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:10:28.627343192Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-05T00:10:28.627368755Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=5
level=info ts=2026-05-05T00:10:28.921204806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:10:28.924931855Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:10:28.924953618Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:10:28.924975215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:10:28.927766109Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:10:30.291332695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:10:31.670088863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:10:31.67340427Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:10:31.785712254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:10:31.789152059Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-05T00:10:31.958651444Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=warn ts=2026-05-05T00:10:35.291033355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:10:35.291083005Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=debug ts=2026-05-05T00:10:38.625168026Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-05T00:10:38.984315966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:10:38.987946813Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:10:39.950792041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:10:39.954244006Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:10:40.29118301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:10:40.670689023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:10:40.674258216Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-05T00:10:42.015498478Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-05T00:10:42.015524524Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=4
level=info ts=2026-05-05T00:10:42.421313783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T00:10:42.425193105Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df52edf97%3A19df52f57bf%3Ab459989b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-05T00:10:42.425216596Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:10:42.425240383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:10:42.428787917Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:10:45.272538146Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:10:45.273324874Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:10:45.290715772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:10:45.290771926Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T00:10:45.290768681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:10:45.290781656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:10:45.291007279Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-05T00:10:47.315174384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:10:47.325179469Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:10:48.086202493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:10:48.091596239Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:10:48.62502641Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=warn ts=2026-05-05T00:10:50.291223802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:10:51.958226256Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-05T00:10:53.279461402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:10:53.283745958Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:10:53.532388545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:10:53.538402815Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:10:53.655798064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:10:53.663344883Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:10:54.419333203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:10:54.422680273Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:10:54.422708468Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:10:54.422733511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:10:54.425528895Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T00:10:55.291632675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:10:55.291900876Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-05T00:10:55.291918259Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-05T00:10:55.559658979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:10:56.38872745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:10:56.392291664Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:10:57.021004714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:10:57.025769817Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:10:57.490311022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:10:57.495045216Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:10:57.887386099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:10:57.891059837Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T00:10:58.624983205Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-05T00:10:59.778402062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:10:59.786616207Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T00:11:00.291122476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:11:00.751305134Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:11:01.958654147Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-05T00:11:02.202922628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:11:02.206420565Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:11:02.206444595Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:11:02.206468608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:11:02.209350619Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:11:04.692054286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:11:04.695480657Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:11:05.290687985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:11:05.2921516Z caller=checkpoint.go:372 msg="writing series" size="463 kB"
level=info ts=2026-05-05T00:11:08.051769001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:11:08.057311685Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T00:11:08.62495324Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-05T00:11:08.624978801Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-05T00:11:09.942964401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:11:09.946779119Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T00:11:10.290858334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:11:11.444321996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:11:11.448713529Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:11:11.958412166Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-05T00:11:15.242733603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:11:15.24725609Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:11:15.290656095Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:11:15.290692436Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T00:11:15.290687876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:11:15.290701685Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:11:15.291822613Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T00:11:15.659018422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:11:15.662613952Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:11:18.625098645Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-05T00:11:20.291173986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:11:21.138324817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:11:21.146173909Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:11:21.95778353Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-05T00:11:22.557510989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:11:22.561496945Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:11:22.56152487Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:11:22.561552486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:11:22.564593413Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:11:24.056410906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:11:24.060615825Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-05T00:11:25.147651693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T00:11:25.291330212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:11:25.291453374Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
ts=2026-05-05T00:11:25.562128559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-05T00:11:28.624591748Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-05T00:11:30.291333197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:11:31.181914859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:11:31.186724477Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:11:31.514813403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:11:31.518288378Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:11:31.518314441Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:11:31.518341719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:11:31.520981312Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:11:31.958865701Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-05T00:11:31.958894149Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=7
level=info ts=2026-05-05T00:11:33.869653703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:11:33.872720528Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-05T00:11:35.291130517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:11:35.291643311Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-05T00:11:35.291658666Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=debug ts=2026-05-05T00:11:35.54406909Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:11:35.54412555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:11:35.547661648Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:11:36.717708421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:11:36.725037782Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:11:38.625439475Z caller=checkpoint.go:372 msg="writing series" size="814 kB"
level=info ts=2026-05-05T00:11:39.060286133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:11:39.063649571Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-05T00:11:40.290816179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:11:40.351028549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:11:40.35946158Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:11:41.957945647Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=info ts=2026-05-05T00:11:42.177604621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:11:42.180704204Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:11:42.813567742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:11:42.816797758Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:11:42.89316188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:11:42.896981011Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:11:44.051125125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:11:44.055203518Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:11:44.055222918Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:11:44.055244697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:11:44.058753766Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T00:11:45.272465312Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:11:45.27351349Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T00:11:45.290737006Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:11:45.290767264Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:11:45.2907776Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:11:45.290809742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:11:45.290924276Z caller=checkpoint.go:372 msg="writing series" size="954 B"
level=info ts=2026-05-05T00:11:46.041624369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:11:46.044670798Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:11:47.440894293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:11:47.444403294Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:11:47.896804922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:11:47.90031899Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:11:48.594075782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:11:48.600547548Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T00:11:48.624633297Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T00:11:49.2742201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:11:49.277557154Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-05T00:11:49.546526994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42448"
level=info ts=2026-05-05T00:11:50.046466573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:11:50.049578596Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T00:11:50.291157538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:11:51.680599494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:11:51.691236542Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:11:51.69125818Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:11:51.691305142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:11:51.694803019Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:11:51.958556205Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
ts=2026-05-05T00:11:54.960245302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54576"
level=warn ts=2026-05-05T00:11:55.291503563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:11:55.296733172Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-05-05T00:11:55.296904982Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.2 MB" series=6
ts=2026-05-05T00:11:55.563854823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:11:56.65786684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="630 B" avg_comp="630 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:11:56.661484095Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:11:57.571311645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:11:57.575790381Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:11:57.680216911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:11:57.685683401Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T00:11:58.624345035Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-05T00:12:00.290765739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:12:01.958467798Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=warn ts=2026-05-05T00:12:05.290676392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:12:05.291897726Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-05T00:12:07.434366108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:12:07.441195367Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:12:07.645805843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:12:07.649385832Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:12:07.674680143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:12:07.678411443Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:12:08.626783696Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T00:12:08.62680536Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=4
level=warn ts=2026-05-05T00:12:10.290628532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:12:10.70287236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:12:10.706787128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T00:12:11.182587225Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:12:11.958641475Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-05T00:12:13.855146929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:12:13.858593133Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:12:15.291061173Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:12:15.291093045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T00:12:15.291085944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:12:15.2911005Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:12:15.291157976Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
ts=2026-05-05T00:12:15.408539197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56672"
level=info ts=2026-05-05T00:12:17.960303671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:12:17.963932072Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-05T00:12:18.625160299Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-05T00:12:19.864472543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:12:19.868993057Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T00:12:19.932523262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T00:12:20.291453451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:12:21.958846506Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
level=info ts=2026-05-05T00:12:22.747410788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:12:22.752667967Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:12:23.815425503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:12:23.818868463Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:12:24.15082409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:12:24.154116578Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:12:25.215272658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:12:25.218751906Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:12:25.218772163Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:12:25.218795137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:12:25.221300067Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-05T00:12:25.291463336Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T00:12:25.291482804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:12:25.565920352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:12:25.935536934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:12:25.938889194Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:12:27.751112745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:12:27.755035619Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:12:28.625320983Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-05T00:12:30.290959882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:12:31.883925907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:12:31.893485385Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:12:31.958606968Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-05T00:12:33.079243931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:12:33.086642522Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:12:35.290671198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:12:35.292305765Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=debug ts=2026-05-05T00:12:35.292321935Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-05-05T00:12:36.449147396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:12:36.453432125Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:12:38.625212304Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-05-05T00:12:40.291433744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:12:40.404708631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:12:40.408301209Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:12:40.569009632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:12:40.572362764Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:12:41.892691763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:12:41.896291307Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:12:41.958444364Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-05T00:12:43.356848563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:12:43.36020358Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:12:43.36022387Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:12:43.360245608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:12:43.363099824Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:12:44.135793893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:12:44.142976663Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T00:12:44.425943162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:12:44.432246917Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:12:44.43229067Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:12:44.432320316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:12:44.438680493Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:12:45.272511886Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:12:45.273183346Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:12:45.291533357Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:12:45.291556987Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:12:45.291564713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:12:45.291591242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:12:45.291867903Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
ts=2026-05-05T00:12:45.410109143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43804"
level=debug ts=2026-05-05T00:12:46.187563403Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:12:48.624318479Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-05T00:12:49.522022881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:12:49.525646846Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T00:12:50.291140396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:12:51.95817618Z caller=checkpoint.go:372 msg="writing series" size="471 kB"
level=debug ts=2026-05-05T00:12:51.958199541Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
ts=2026-05-05T00:12:54.96351274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36686"
level=warn ts=2026-05-05T00:12:55.290878499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:12:55.290980678Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-05T00:12:55.397242343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:12:55.403669059Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-05T00:12:55.567649938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:12:56.37252594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:12:56.375736518Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-05T00:12:56.499690896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:12:56.751322032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:12:56.757841806Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:12:56.75786402Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:12:56.75788713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:12:56.763258293Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:12:58.624823412Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=warn ts=2026-05-05T00:13:00.291466661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:13:01.771367721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:13:01.774885035Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:13:01.95839017Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-05T00:13:04.965504191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:13:04.970635092Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-05T00:13:05.291400842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:13:05.291518902Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=info ts=2026-05-05T00:13:07.656679049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:13:07.663802538Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:13:08.350971394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:13:08.354348739Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:13:08.625248911Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-05T00:13:09.961280185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:13:09.964804234Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-05T00:13:10.291534423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:13:10.627107909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:13:10.630763106Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:13:10.630785031Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:13:10.630822554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:13:10.633835222Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T00:13:11.958885231Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T00:13:11.958907482Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=6
level=info ts=2026-05-05T00:13:13.213840744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:13:13.217331305Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:13:14.153056354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:13:14.157125318Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:13:15.290827946Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:13:15.290867649Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:13:15.290877402Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:13:15.290877209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:13:15.290920834Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T00:13:16.818400018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:13:16.822864826Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:13:18.396489988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:13:18.402810372Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:13:18.463388881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:13:18.466672499Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-05T00:13:18.624347645Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=warn ts=2026-05-05T00:13:20.291278869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:13:21.020641305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:13:21.958330359Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-05T00:13:22.461834073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:13:22.465530182Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:13:22.795468537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:13:22.802769436Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T00:13:25.290615994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:13:25.291758481Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-05T00:13:25.56962504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:13:26.712677115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:13:26.719390138Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T00:13:28.625366367Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-05T00:13:29.861744105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:13:29.865503739Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:13:30.261824931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:13:30.268430411Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:13:30.291938834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:13:30.417283786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:13:30.421247638Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:13:31.936817505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:13:31.94318935Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:13:31.960604553Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-05T00:13:31.960622731Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=6
level=info ts=2026-05-05T00:13:34.943813238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:13:34.950282796Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T00:13:35.291352347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:13:35.291475357Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-05T00:13:35.82812807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:13:35.831800051Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:13:38.624798085Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=info ts=2026-05-05T00:13:39.183460287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:13:39.18661056Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:13:39.387474798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:13:39.3908562Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=warn ts=2026-05-05T00:13:40.290929415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:13:41.958535078Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-05T00:13:42.665600483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="804 B" avg_comp="804 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:13:42.66857228Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:13:45.272516709Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:13:45.272600378Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:13:45.290853194Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:13:45.290876138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-05T00:13:45.290902072Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-05T00:13:45.290911516Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:13:45.290924779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:13:48.624493195Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-05T00:13:50.291328161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:13:50.955280929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:13:50.959393433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:13:51.895235298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:13:51.899036187Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-05T00:13:51.958461768Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-05T00:13:51.958480746Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-05T00:13:53.286147101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:13:53.289734768Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-05T00:13:53.820305311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:13:53.820376927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="494 kB" avg_comp="494 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" idle=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:13:53.83039003Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=warn ts=2026-05-05T00:13:55.290673734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:13:55.291982076Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
ts=2026-05-05T00:13:55.571715907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:13:57.878119305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:13:57.879610541Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T00:13:58.625392472Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=info ts=2026-05-05T00:13:58.759532263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:13:58.763362587Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:14:00.098292164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:14:00.101691075Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:14:00.199920734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:14:00.203714492Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:14:00.203733634Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:14:00.203754083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:14:00.20709029Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T00:14:00.291197595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:14:01.957625381Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-05T00:14:02.586563922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:14:02.589927663Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:14:02.803491434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:14:02.806795258Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T00:14:05.291423043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:14:05.29142483Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-05T00:14:05.743389436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:14:05.747493759Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=info ts=2026-05-05T00:14:07.111601291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:14:07.116417355Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:14:08.401664641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:14:08.408710624Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:14:08.627683576Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T00:14:08.627714488Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=5
level=warn ts=2026-05-05T00:14:10.291011481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:14:10.428455294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:14:10.432943697Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:14:11.958217268Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=info ts=2026-05-05T00:14:14.750165021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:14:14.75491589Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:14:15.291143882Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:14:15.291170283Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-05T00:14:15.291164689Z caller=checkpoint.go:384 msg="flushing series" totalSize="212 kB" series=1
level=info ts=2026-05-05T00:14:15.291180652Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:14:15.291194058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:14:15.295819449Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-05T00:14:15.29720317Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006351395s
level=info ts=2026-05-05T00:14:18.164646311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:14:18.171032168Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:14:18.171055621Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:14:18.171083226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:14:18.177321934Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:14:18.722314454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:14:18.726171722Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T00:14:20.291445925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:14:21.143385283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:14:21.149121413Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df52f479e%3A19df55e2df5%3A68788a3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T00:14:21.149145477Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:14:21.149173242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:14:21.152693723Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-05T00:14:22.526939837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:14:22.530977116Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:14:22.531004273Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:14:22.531028796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:14:22.534617939Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T00:14:23.285302491Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T00:14:24.967914179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53012"
level=warn ts=2026-05-05T00:14:25.291289201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:14:25.573522784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:14:26.549903316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:14:26.553408829Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:14:28.683384274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:14:28.68943282Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:14:29.194800253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:14:29.201295522Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:14:30.291164556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:14:30.365401428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:14:30.368540592Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:14:32.167059643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:14:32.170662234Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-05T00:14:32.747313136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:14:32.754190071Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:14:34.337310464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:14:34.340778224Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:14:35.29140577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:14:35.842959967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:14:35.846217573Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:14:36.487761712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:14:36.491049356Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-05T00:14:39.537793076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:14:39.578978457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:14:39.582645712Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=warn ts=2026-05-05T00:14:40.291453745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:14:40.764974172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:14:40.76824568Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:14:45.104030969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:14:45.108218329Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:14:45.272506528Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:14:45.272536983Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T00:14:45.290790744Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-05T00:14:45.290806908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:14:45.290828901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:14:45.290841353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:14:45.290853662Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T00:14:45.290946135Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-05T00:14:45.290955051Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T00:14:45.290975402Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.689µs success=true
level=debug ts=2026-05-05T00:14:45.291173804Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-05T00:14:45.418598499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43188"
level=debug ts=2026-05-05T00:14:48.624650635Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-05T00:14:48.66473086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:14:48.668675577Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:14:50.291315499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:14:50.832741845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:14:50.836802414Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-05T00:14:51.958251911Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-05T00:14:53.35141897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:14:53.355340284Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-05T00:14:54.96978568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:32930"
level=warn ts=2026-05-05T00:14:55.290910292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:14:55.292029427Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-05-05T00:14:55.575323413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:14:55.692308268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:14:55.695575376Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:14:57.561340385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="173 kB" avg_comp="173 kB" total_uncomp="890 kB" avg_uncomp="890 kB" idle=1 labels="{stream=\"integration2-lvtd\"}"
level=debug ts=2026-05-05T00:14:57.561392272Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-05T00:14:57.56653468Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=info ts=2026-05-05T00:14:58.473429861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:14:58.480644978Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:14:58.566301447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:14:58.569209494Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:14:58.625542406Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-05T00:14:59.28052356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:14:59.284420306Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-05T00:15:00.290953584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:15:00.641456761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:15:00.64787476Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:15:01.958753554Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T00:15:02.694208458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:15:02.697848055Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:15:04.177968347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:15:04.181061208Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:15:04.220125541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:15:04.223404327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T00:15:05.290917281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:15:05.292136741Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-05T00:15:06.053389151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:15:06.057422192Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:15:08.637035335Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-05T00:15:08.637057968Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=info ts=2026-05-05T00:15:10.012377261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="890 kB" avg_uncomp="890 kB" full=1 labels="{stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:15:10.017250926Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=warn ts=2026-05-05T00:15:10.291308887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:15:11.971504097Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-05T00:15:11.97153117Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-05-05T00:15:12.180678313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:15:12.185343018Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:15:15.291467359Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:15:15.291490437Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:15:15.291498377Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:15:15.291491791Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-05T00:15:15.291500003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:15:15.420092507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38426"
level=info ts=2026-05-05T00:15:15.521733512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:15:15.52495423Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:15:16.537716639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:15:16.541487684Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:15:17.435205475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:15:17.43864726Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:15:18.625400414Z caller=checkpoint.go:372 msg="writing series" size="475 kB"
level=warn ts=2026-05-05T00:15:20.291214126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:15:21.243217733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:15:21.24957397Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:15:21.958065051Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-05T00:15:25.066153299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:15:25.069583089Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-05T00:15:25.291625617Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-05T00:15:25.291632415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:15:25.576863627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:15:26.874018634Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:15:27.569084547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:15:27.576706639Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T00:15:28.626697364Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-05T00:15:28.626734642Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=5
level=warn ts=2026-05-05T00:15:30.290950076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:15:31.22038998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:15:31.225011226Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=debug ts=2026-05-05T00:15:31.958259194Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-05T00:15:32.505854335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:15:32.509360909Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:15:34.199175041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:15:34.203155725Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:15:34.560288993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:15:34.563755081Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:15:35.290800524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:15:35.292004628Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-05T00:15:36.850093804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:15:36.854458048Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:15:38.624951438Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-05T00:15:39.666202422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T00:15:40.29092077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:15:40.486375368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="890 kB" avg_uncomp="890 kB" full=1 labels="{stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:15:40.490278978Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=info ts=2026-05-05T00:15:40.589534609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:15:40.593516819Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:15:42.002699219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:15:42.006487869Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:15:42.014872034Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-05T00:15:42.014889062Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=4
level=info ts=2026-05-05T00:15:42.613416835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:15:42.616768873Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:15:42.726155998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:15:42.730470042Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-05T00:15:45.272491135Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:15:45.273545351Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:15:45.273723063Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-05T00:15:45.273766348Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975488 n=1
level=debug ts=2026-05-05T00:15:45.273807289Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975488 n=1
level=info ts=2026-05-05T00:15:45.290761868Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:15:45.290782994Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:15:45.290791981Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:15:45.290901496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:15:45.291994029Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-05T00:15:46.749945452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:15:46.753488019Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:15:47.123636761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:15:47.129133492Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:15:47.235498942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:15:47.238775294Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:15:48.625832157Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=info ts=2026-05-05T00:15:50.086314451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:15:50.095414925Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:15:50.095449049Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:15:50.095484636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:15:50.107466643Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T00:15:50.291599501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:15:51.958010648Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-05T00:15:53.901215093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:15:53.905122848Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:15:53.995637381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:15:54.003368647Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:15:55.291063685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:15:55.292312077Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
ts=2026-05-05T00:15:55.578348042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:15:58.625003379Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=debug ts=2026-05-05T00:15:59.539037323Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T00:16:00.291022406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:16:01.95873651Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-05T00:16:01.958759103Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-05-05T00:16:03.649127669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:16:03.653195407Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:16:04.091602685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:16:04.098696088Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:16:04.110755916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:16:04.118782703Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:16:05.269843397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:16:05.274521444Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T00:16:05.290734616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:16:05.292211988Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T00:16:08.625239062Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-05T00:16:09.763291743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:16:09.770809881Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:16:10.065400649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:16:10.068440572Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T00:16:10.291077106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:16:10.44593741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:16:10.449474619Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:16:11.958372901Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-05T00:16:15.291505675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:16:15.291534279Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:16:15.291541909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:16:15.291549593Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-05T00:16:15.291560167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:16:15.423105993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:32790"
level=info ts=2026-05-05T00:16:16.054518832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:16:16.05819891Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:16:18.00506034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:16:18.009028217Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=debug ts=2026-05-05T00:16:18.624954545Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-05T00:16:20.291033072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:16:21.572106794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:16:21.577246673Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:16:21.958507471Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T00:16:21.958528617Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-05T00:16:22.217012238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:16:22.220490606Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:16:22.837236494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:16:22.840400907Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:16:22.951815787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:16:22.955688191Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:16:23.421070364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:16:23.425061843Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-05T00:16:23.846279429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:16:23.849616257Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:16:23.849640823Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:16:23.849662389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:16:23.852246064Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-05T00:16:25.290837673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:16:25.292088774Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
ts=2026-05-05T00:16:25.579420846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:16:27.463440412Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:16:28.477615063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:16:28.482333935Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:16:28.624891744Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-05T00:16:29.830103791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:16:29.833408028Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-05T00:16:30.29162113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:16:30.731706668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:16:30.735675639Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:16:31.959234869Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-05T00:16:31.959258563Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-05T00:16:32.981300332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:16:32.989023772Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:16:34.767604147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:16:34.772052867Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T00:16:35.29151714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:16:35.292365736Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-05T00:16:35.292381675Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-05T00:16:35.774834864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="890 kB" avg_uncomp="890 kB" full=1 labels="{stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:16:35.778390733Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=info ts=2026-05-05T00:16:36.762498852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:16:36.76852078Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:16:37.953878595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:16:38.625401124Z caller=checkpoint.go:372 msg="writing series" size="824 kB"
level=info ts=2026-05-05T00:16:40.070145876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:16:40.073765704Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T00:16:40.29108717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:16:41.958595334Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-05T00:16:43.512199113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:16:43.515454468Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:16:45.27251546Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:16:45.272949546Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-05T00:16:45.291292618Z caller=checkpoint.go:372 msg="writing series" size="954 B"
level=info ts=2026-05-05T00:16:45.291306007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T00:16:45.291318533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:16:45.291329218Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:16:45.291337058Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T00:16:45.424943032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38070"
level=info ts=2026-05-05T00:16:45.432415906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:16:45.436082193Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T00:16:48.624928687Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T00:16:50.291001712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:16:51.885504209Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:16:51.958649562Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
ts=2026-05-05T00:16:54.976807836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45260"
level=warn ts=2026-05-05T00:16:55.291204127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:16:55.296444534Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-05T00:16:55.296480528Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.0 MB" series=6
ts=2026-05-05T00:16:55.581777675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:16:57.428020513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:16:57.436401068Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:16:58.252521007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:16:58.256417715Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:16:58.256442931Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:16:58.256465301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:16:58.259153724Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:16:58.625128959Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-05T00:16:58.898774538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:16:58.902447622Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:16:59.3147666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:16:59.31948158Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T00:17:00.290630831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:17:00.4200099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:17:00.423704015Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:17:00.53155077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:17:00.537989391Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:17:01.958954867Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-05T00:17:05.290736384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:17:05.292054064Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-05T00:17:07.066169389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="630 B" avg_comp="630 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:17:07.070522912Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:17:08.626842342Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T00:17:08.626864429Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=4
level=warn ts=2026-05-05T00:17:10.290925029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:17:10.501929038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:17:10.505616963Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-05T00:17:11.958666638Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-05T00:17:12.114123626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:17:12.11798225Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:17:12.874607201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:17:12.878046933Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:17:15.131381124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:17:15.134684208Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:17:15.238567568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:17:15.241746452Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:17:15.241767487Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:17:15.24178849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:17:15.244382771Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:17:15.290530321Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:17:15.290552836Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:17:15.290561043Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:17:15.290602027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:17:15.291666024Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-05T00:17:15.556245607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:17:15.559281744Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:17:16.103035004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:17:16.107107769Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=info ts=2026-05-05T00:17:17.227211959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:17:17.230771243Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:17:17.230789876Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:17:17.230811327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:17:17.233487836Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:17:17.447784294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:17:17.450942739Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:17:18.376793343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:17:18.381547125Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:17:18.381569262Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:17:18.381592445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:17:18.385140276Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-05T00:17:18.6250679Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-05T00:17:18.815684541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:17:18.820117461Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T00:17:19.942760111Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T00:17:20.29150527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:17:21.830014118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:17:21.833834199Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:17:21.958526739Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
level=info ts=2026-05-05T00:17:22.594307518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="890 kB" avg_uncomp="890 kB" full=1 labels="{stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:17:22.598031786Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=info ts=2026-05-05T00:17:22.85011943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:17:22.856297998Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:17:24.184354799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T00:17:25.290806423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:17:25.291901319Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-05T00:17:25.583931729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:17:25.811071602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:17:25.818762922Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:17:27.898215191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:17:27.901645661Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:17:28.624706574Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-05T00:17:30.291225895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:17:31.958182131Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-05T00:17:33.685397486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:17:33.692624205Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:17:34.196104801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:17:34.200169293Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:17:34.48612505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:17:34.489692045Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T00:17:35.291014907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:17:35.2924826Z caller=checkpoint.go:372 msg="writing series" size="557 kB"
level=debug ts=2026-05-05T00:17:35.292501473Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-05-05T00:17:37.858974982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:17:37.863636435Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:17:38.625289489Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-05T00:17:39.074682582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:17:39.080679373Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T00:17:40.29131843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:17:41.958153719Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-05T00:17:44.041072074Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:17:45.27261425Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:17:45.273053918Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:17:45.291366978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:17:45.291388616Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:17:45.291397176Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:17:45.291445416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:17:45.291709571Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
ts=2026-05-05T00:17:45.427121167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40818"
level=debug ts=2026-05-05T00:17:48.625032513Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-05T00:17:48.897108524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:17:48.901582722Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T00:17:50.291527937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:17:51.958334364Z caller=checkpoint.go:372 msg="writing series" size="476 kB"
level=debug ts=2026-05-05T00:17:51.958359445Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-05T00:17:52.088684413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:17:52.093340295Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:17:53.899223492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:17:53.902997881Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:17:54.443468207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:17:54.449612982Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-05T00:17:55.290796416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:17:55.291960176Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-05T00:17:55.585657772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:17:55.723991698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:17:55.727190036Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:17:55.727209619Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:17:55.727228878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:17:55.730475981Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:17:57.104704878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:17:57.108196019Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:17:57.108218349Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:17:57.108240868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:17:57.111107334Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:17:57.546316423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:17:57.54939894Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:17:58.625942193Z caller=checkpoint.go:372 msg="writing series" size="537 kB"
level=warn ts=2026-05-05T00:18:00.291480159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:18:00.83963944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:18:00.842942073Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:18:01.71558835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:18:01.719707606Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=debug ts=2026-05-05T00:18:01.958450662Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=debug ts=2026-05-05T00:18:04.578855643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:18:04.580378649Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T00:18:05.140963327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:18:05.144742323Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-05T00:18:05.291227114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:18:05.291356264Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=info ts=2026-05-05T00:18:06.510117002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:18:06.517782247Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:18:06.966602375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:18:06.969676192Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:18:06.998733919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:18:07.001483104Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:18:07.194025145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:18:07.197852326Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:18:07.328522108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="890 kB" avg_uncomp="890 kB" full=1 labels="{stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:18:07.332022266Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=debug ts=2026-05-05T00:18:08.625327379Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-05T00:18:08.625353112Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T00:18:09.852960783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:18:09.856831298Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:18:09.856851643Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:18:09.856886414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:18:09.860047604Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T00:18:10.291146871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:18:10.408459839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:18:10.412398097Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:18:10.768096161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:18:10.771463821Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:18:11.128699257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:18:11.131971772Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:18:11.959216966Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T00:18:11.959240728Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-05-05T00:18:12.084815944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:18:12.088139337Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:18:15.291293712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:18:15.291317096Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:18:15.291325301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:18:15.29132064Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-05T00:18:15.291328568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:18:15.429053891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40996"
level=info ts=2026-05-05T00:18:16.375742798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:18:16.379179868Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:18:16.379199772Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:18:16.379220516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:18:16.381918269Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:18:17.798492936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:18:17.807019249Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:18:18.624691639Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=debug ts=2026-05-05T00:18:19.477493242Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T00:18:19.5745569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:36226"
level=warn ts=2026-05-05T00:18:20.291170791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:18:21.95864779Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-05T00:18:24.008017453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:18:24.012986577Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:18:24.253319001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:18:24.259226458Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:18:25.252286864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:18:25.256461089Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T00:18:25.291530476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:18:25.291587887Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-05T00:18:25.587225231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:18:28.601945599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:18:28.605456975Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:18:28.624673654Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=warn ts=2026-05-05T00:18:30.290957555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:18:31.959984787Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-05T00:18:31.960010781Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=6
level=warn ts=2026-05-05T00:18:35.291151933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:18:35.29134551Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-05T00:18:35.97339455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:18:35.977128576Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:18:35.977152395Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:18:35.977174995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:18:35.980096947Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:18:38.624752773Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=info ts=2026-05-05T00:18:39.718845903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:18:39.72333795Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:18:40.291187027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:18:41.95804239Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-05T00:18:42.869718641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:18:42.87342988Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:18:44.446687975Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:18:45.019384144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="804 B" avg_comp="804 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:18:45.022882538Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:18:45.272627996Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:18:45.273004714Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:18:45.291682424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:18:45.291713247Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:18:45.29172197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:18:45.291780564Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-05T00:18:45.291736497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:18:46.052107272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:18:46.055349039Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:18:46.324938761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:18:46.328137538Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:18:47.50962894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:18:47.513458592Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-05T00:18:47.671037278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:18:47.679618452Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:18:47.747710484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:18:47.754941158Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:18:48.198374123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:18:48.201633684Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:18:48.625578262Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
ts=2026-05-05T00:18:49.576307625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53330"
level=info ts=2026-05-05T00:18:49.734209424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:18:49.737425448Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T00:18:50.290609178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:18:51.369533442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:18:51.372813204Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:18:51.607592125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:18:51.611852699Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:18:51.611874566Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:18:51.611901361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:18:51.615546227Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T00:18:51.959352021Z caller=checkpoint.go:372 msg="writing series" size="534 kB"
level=debug ts=2026-05-05T00:18:51.95937473Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-05-05T00:18:52.563333033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:18:52.809013953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:18:52.813082464Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=info ts=2026-05-05T00:18:53.002638579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:18:53.006328791Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:18:53.244970185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:18:53.248605917Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:18:53.621917888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:18:53.62506612Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:18:55.109239329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:18:55.113171958Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:18:55.113199427Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:18:55.113225328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:18:55.11615783Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-05T00:18:55.290614375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:18:55.291961069Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
ts=2026-05-05T00:18:55.588363856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:18:58.625955954Z caller=checkpoint.go:372 msg="writing series" size="644 kB"
level=warn ts=2026-05-05T00:19:00.290611383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:19:00.976992351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="890 kB" avg_uncomp="890 kB" full=1 labels="{stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:19:00.980814407Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=debug ts=2026-05-05T00:19:01.958401313Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-05T00:19:03.225317616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:19:03.228785736Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:19:03.97498338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:19:03.979432954Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T00:19:05.291178433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:19:05.291204662Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-05T00:19:07.539187409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:19:07.54357651Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:19:08.108219319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:19:08.115826452Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:19:08.627132184Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-05T00:19:08.627154114Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=warn ts=2026-05-05T00:19:10.291225862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:19:10.738079673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:19:10.74128803Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-05T00:19:11.9585724Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=info ts=2026-05-05T00:19:13.690806194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:19:13.695054943Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:19:14.669703264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:19:14.675676617Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:19:14.675696962Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:19:14.675727885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:19:14.681567854Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:19:15.291017538Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:19:15.291045442Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:19:15.291053512Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:19:15.291073762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:19:15.292098507Z caller=checkpoint.go:384 msg="flushing series" totalSize="236 kB" series=1
level=info ts=2026-05-05T00:19:15.300238573Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-05T00:19:15.301602378Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010479286s
ts=2026-05-05T00:19:15.433035327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34402"
level=info ts=2026-05-05T00:19:15.476889363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:19:15.480189171Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:19:17.300450603Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T00:19:20.290848894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:19:24.985432321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51122"
level=warn ts=2026-05-05T00:19:25.291460714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:19:25.434802761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:19:25.442812943Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-05T00:19:25.590150043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:19:26.772143822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:19:26.776318594Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:19:26.776344554Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:19:26.776399619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:19:26.779976669Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:19:28.230955755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:19:28.237105567Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:19:28.23712602Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:19:28.237148645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:19:28.24255192Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:19:29.865082202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:19:29.86828522Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T00:19:30.291232224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:19:31.376737675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:19:31.59933345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:19:31.606409794Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:19:32.670339877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:19:32.674728815Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:19:33.327462055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:19:33.331542055Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=info ts=2026-05-05T00:19:35.170007708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:19:35.174204943Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-05T00:19:35.291595539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:19:36.68666916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:19:36.691227645Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:19:37.603881287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:19:37.60725433Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:19:38.088744335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:19:38.092482944Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:19:38.807329374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:19:38.810684923Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T00:19:40.290952724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:19:42.110690938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:19:42.115994407Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:19:42.19722372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:19:42.201112668Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-05T00:19:43.198938866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:19:43.202071305Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:19:43.422819518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:19:43.425784365Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:19:44.56001219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:19:44.563129346Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:19:44.563148707Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:19:44.563169846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:19:44.565579194Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:19:45.272133046Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:19:45.273180505Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T00:19:45.290441598Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T00:19:45.290588986Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:19:45.290607221Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:19:45.290614426Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T00:19:45.290611586Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-05T00:19:45.290625836Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=warn ts=2026-05-05T00:19:45.290635228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:19:45.290668961Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=27.291µs success=true
level=debug ts=2026-05-05T00:19:45.290855058Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-05T00:19:45.434478256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35682"
level=debug ts=2026-05-05T00:19:45.584027239Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:19:45.584036879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:19:45.587775441Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T00:19:46.417983456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:19:46.424791394Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:19:48.624840558Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-05T00:19:49.1102128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:19:49.114318464Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:19:49.485389343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:19:49.488557298Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-05T00:19:50.290612326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:19:51.958038478Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-05T00:19:54.296852094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="890 kB" avg_uncomp="890 kB" full=1 labels="{stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:19:54.300440528Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=debug ts=2026-05-05T00:19:55.291427501Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T00:19:55.291441554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:19:55.591293823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:19:55.976242028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:19:55.980140762Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T00:19:58.625217747Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=info ts=2026-05-05T00:19:59.515596979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:19:59.519210241Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=warn ts=2026-05-05T00:20:00.290925279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:20:00.98053672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:20:00.984454911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:20:00.987533845Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:20:00.991011893Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:20:01.957773351Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T00:20:01.972006045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:20:01.975309486Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:20:02.090673102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:20:02.094361121Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T00:20:05.291225318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:20:05.291296367Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=debug ts=2026-05-05T00:20:08.639570226Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-05T00:20:08.639595964Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=info ts=2026-05-05T00:20:09.857135326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:20:09.861324677Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T00:20:10.290965701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:20:11.966912706Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-05T00:20:11.966937395Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-05-05T00:20:13.073938455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2cb3d97231fbb42c immediate=false num_chunks=1 total_comp="586 kB" avg_comp="586 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" idle=1 labels="{stream=\"build-e2e-lrta\"}"
level=debug ts=2026-05-05T00:20:13.073992404Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-05T00:20:13.086102029Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2cb3d97231fbb42c/19df538ea33%3A19df54aa544%3Ab0e5845b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lrta\"}"
level=info ts=2026-05-05T00:20:13.107195545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:20:13.110625072Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:20:14.527629948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:20:14.535820786Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:20:15.291184322Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:20:15.291211219Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:20:15.291219462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:20:15.291218237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:20:15.291224546Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-05-05T00:20:15.436125982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52148"
level=debug ts=2026-05-05T00:20:18.624760428Z caller=checkpoint.go:372 msg="writing series" size="444 kB"
level=info ts=2026-05-05T00:20:19.201172099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:20:19.205408233Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T00:20:20.291358411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:20:21.535194862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:20:21.539253397Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T00:20:21.958424371Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-05T00:20:22.211248588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:20:22.21831083Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:20:24.463098817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:20:24.466595843Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:20:24.597995844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:20:24.601690909Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:20:24.601712122Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:20:24.601734873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:20:24.60520305Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-05T00:20:25.291136795Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-05T00:20:25.291156338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:20:25.592702252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:20:26.859023508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:20:26.862464189Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:20:28.296702475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:20:28.340704904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:20:28.343845111Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:20:28.517544785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:20:28.52298222Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:20:28.523007231Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:20:28.523037271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:20:28.527026171Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=debug ts=2026-05-05T00:20:28.628548403Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-05T00:20:28.628569856Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=5
level=info ts=2026-05-05T00:20:28.853387247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:20:28.856591381Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:20:29.866473041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:20:29.869970486Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:20:30.101608343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:20:30.104825053Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T00:20:30.291455557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:20:31.532833398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2cb3d97231fbb42c immediate=false num_chunks=1 total_comp="601 kB" avg_comp="601 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lrta\"}"
level=error ts=2026-05-05T00:20:31.537695162Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2cb3d97231fbb42c/19df538ea33%3A19df54aa544%3Ab0e5845b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lrta\"}"
level=debug ts=2026-05-05T00:20:31.957835936Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-05T00:20:32.187386312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:20:32.19212719Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T00:20:35.291128627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:20:35.291311881Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-05T00:20:36.525149852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:20:36.528496104Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:20:38.107984865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:20:38.111856262Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-05T00:20:38.625104298Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-05T00:20:40.290848925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:20:40.713388948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:20:40.718483874Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:20:41.569000179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:20:41.572879708Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-05T00:20:42.011943864Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-05T00:20:42.011973932Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=4
level=info ts=2026-05-05T00:20:43.114170877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:20:43.119233987Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:20:43.119277764Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:20:43.119300281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:20:43.122037596Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T00:20:43.534435386Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:20:45.272611015Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:20:45.272801577Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:20:45.29113433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:20:45.291153016Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:20:45.291161176Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:20:45.291220158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:20:45.291294728Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-05T00:20:45.662130086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:20:45.665352798Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:20:46.287907688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:20:46.295700823Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=info ts=2026-05-05T00:20:46.326896918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:20:46.330750998Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:20:46.330776235Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:20:46.330800404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:20:46.336362563Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:20:48.628141866Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
ts=2026-05-05T00:20:49.584649914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48458"
level=warn ts=2026-05-05T00:20:50.291144639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:20:51.958373216Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-05T00:20:53.158590597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="890 kB" avg_uncomp="890 kB" full=1 labels="{stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:20:53.165057502Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=info ts=2026-05-05T00:20:53.82640459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:20:53.833242545Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:20:54.234571127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:20:54.239701137Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T00:20:55.291944521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:20:55.292288478Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
ts=2026-05-05T00:20:55.593696558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:20:56.083224474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:20:56.089825257Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:20:57.321611076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:20:57.328652432Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:20:57.328674699Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:20:57.328698288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:20:57.335671766Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:20:58.624852846Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-05T00:21:00.290955371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:21:01.958826163Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-05T00:21:01.95885397Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-05T00:21:02.39280624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:21:02.396326327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T00:21:05.291137138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:21:05.291417313Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=info ts=2026-05-05T00:21:05.721757049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:21:05.726664432Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:21:06.102983752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:21:06.106983967Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:21:06.604378051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:21:06.608388969Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:21:07.114772898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:21:07.118310317Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:21:07.86130637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:21:07.865352202Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T00:21:08.624820156Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-05T00:21:09.054706586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:21:09.058993984Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=info ts=2026-05-05T00:21:09.89577589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:21:09.902840511Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:21:10.291531119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:21:11.063168389Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:21:11.085529427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2cb3d97231fbb42c immediate=false num_chunks=1 total_comp="601 kB" avg_comp="601 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lrta\"}"
level=error ts=2026-05-05T00:21:11.090428524Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2cb3d97231fbb42c/19df538ea33%3A19df54aa544%3Ab0e5845b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lrta\"}"
level=debug ts=2026-05-05T00:21:11.957732102Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-05T00:21:14.011863239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:21:14.020187659Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:21:14.12052032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:21:14.1239711Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:21:15.291526557Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:21:15.291553536Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:21:15.291562599Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:21:15.29158814Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-05T00:21:15.291589398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:21:16.259533456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:21:16.2657684Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:21:16.904704365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:21:16.90840099Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:21:18.624941677Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-05T00:21:18.793323899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:21:18.797177344Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:21:19.707760124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:21:19.711987753Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-05T00:21:20.291509868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:21:20.907759151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:21:21.484180161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:21:21.488500919Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:21:21.755312109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:21:21.758532849Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:21:21.958249725Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T00:21:21.958286859Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-05T00:21:25.29089131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:21:25.291070179Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
ts=2026-05-05T00:21:25.59543202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:21:26.997405997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:21:27.000709191Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:21:28.624467871Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-05T00:21:28.687795187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:21:28.69110856Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:21:30.290938244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:21:31.958358345Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-05T00:21:31.958382139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-05T00:21:33.003146935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:21:33.010056278Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:21:34.121224903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:21:34.124638901Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T00:21:35.290686585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:21:35.292491869Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-05T00:21:35.292505853Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-05T00:21:36.852394037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:21:36.855703882Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-05T00:21:38.625473812Z caller=checkpoint.go:372 msg="writing series" size="869 kB"
level=info ts=2026-05-05T00:21:39.591224579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:21:39.5947767Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-05T00:21:40.291640979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:21:40.429011556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:21:40.433253917Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:21:41.958434937Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=info ts=2026-05-05T00:21:42.00155903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:21:42.00960509Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T00:21:43.998220844Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:21:43.99831768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=278af8730b02432a immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="185 kB" avg_uncomp="185 kB" idle=1 labels="{stream=\"build-e2e-xrqf\"}"
level=error ts=2026-05-05T00:21:44.002641829Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/278af8730b02432a/19df53734c7%3A19df54c2041%3A37ce0b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xrqf\"}"
level=info ts=2026-05-05T00:21:44.880160883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6b319fb3f337eb45 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="890 kB" avg_uncomp="890 kB" full=1 labels="{stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:21:44.88393474Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6b319fb3f337eb45/19df541a174%3A19df545ac9e%3Ad721a312?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-lvtd\"}"
level=error ts=2026-05-05T00:21:44.883960136Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:21:44.883982424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=09cac0c115a46ea5 immediate=false num_chunks=1 total_comp="423 kB" avg_comp="423 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" idle=1 labels="{stream=\"build-e2e-hswr\"}"
level=error ts=2026-05-05T00:21:44.89369791Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9cac0c115a46ea5/19df53855d8%3A19df546a201%3A6c842155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hswr\"}"
level=debug ts=2026-05-05T00:21:45.272233834Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:21:45.273291231Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T00:21:45.290575844Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:21:45.290604123Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:21:45.290614751Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:21:45.290652657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:21:45.291764793Z caller=checkpoint.go:372 msg="writing series" size="954 B"
level=info ts=2026-05-05T00:21:47.586541852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:21:47.591548388Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:21:48.624891128Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T00:21:50.290869624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:21:51.693221353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:21:51.696894947Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T00:21:51.958511555Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-05T00:21:51.958536534Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T00:21:53.684898374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2cb3d97231fbb42c immediate=false num_chunks=1 total_comp="601 kB" avg_comp="601 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lrta\"}"
level=error ts=2026-05-05T00:21:53.68899842Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2cb3d97231fbb42c/19df538ea33%3A19df54aa544%3Ab0e5845b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lrta\"}"
level=info ts=2026-05-05T00:21:55.164070883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:21:55.168387381Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T00:21:55.227552726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:21:55.23063344Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-05T00:21:55.29080666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:21:55.295902317Z caller=checkpoint.go:372 msg="writing series" size="4.2 MB"
level=debug ts=2026-05-05T00:21:55.295923124Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=1
ts=2026-05-05T00:21:55.597119743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:21:56.786689809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:21:56.794805446Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:21:56.794830787Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:21:56.794864119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:21:56.801684345Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:21:57.180839537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=278af8730b02432a immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-xrqf\"}"
level=error ts=2026-05-05T00:21:57.184325862Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/278af8730b02432a/19df53734c7%3A19df54c2041%3A37ce0b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xrqf\"}"
level=info ts=2026-05-05T00:21:57.697939313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:21:57.705153432Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:21:57.920714218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:21:57.924403185Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:21:58.6250176Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-05T00:21:59.767368495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=09cac0c115a46ea5 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"build-e2e-hswr\"}"
level=error ts=2026-05-05T00:21:59.771561191Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9cac0c115a46ea5/19df53855d8%3A19df546a201%3A6c842155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hswr\"}"
level=warn ts=2026-05-05T00:22:00.290993545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:22:01.499656999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:22:01.503233769Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:22:01.958573558Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-05T00:22:02.139050636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:22:02.143372335Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:22:04.536623728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:22:04.540057473Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:22:05.148525359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:22:05.153755842Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=warn ts=2026-05-05T00:22:05.291382036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:22:05.291587046Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-05T00:22:06.061316502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:22:06.06582373Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:22:08.406548765Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:22:08.629451214Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-05T00:22:08.629490717Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=4
level=info ts=2026-05-05T00:22:09.197235676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:22:09.20100844Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:22:10.291575535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:22:11.68669659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:22:11.690081739Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-05T00:22:11.957750999Z caller=checkpoint.go:372 msg="writing series" size="428 kB"
level=info ts=2026-05-05T00:22:12.298552154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:22:12.302884805Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:22:13.921216945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:22:13.928921751Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:22:14.127554219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:22:14.129184555Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T00:22:15.29098639Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:22:15.291015522Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T00:22:15.291007954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:22:15.291023945Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:22:15.291082264Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=info ts=2026-05-05T00:22:16.145533183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:22:16.148779834Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:22:16.705549238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:22:16.709343466Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-05T00:22:18.625004927Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-05T00:22:20.291511958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:22:21.958108481Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
level=info ts=2026-05-05T00:22:22.42431694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:22:22.430992821Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:22:25.290693538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:22:25.291860357Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-05T00:22:25.598721868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:22:26.145824569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="630 B" avg_comp="630 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:22:26.149508972Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=debug ts=2026-05-05T00:22:28.624540929Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-05T00:22:28.843052136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=09cac0c115a46ea5 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"build-e2e-hswr\"}"
level=error ts=2026-05-05T00:22:28.847418295Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9cac0c115a46ea5/19df53855d8%3A19df546a201%3A6c842155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hswr\"}"
level=warn ts=2026-05-05T00:22:30.29144751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:22:30.718641314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:22:30.725165223Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:22:30.823594284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:22:30.827484338Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:22:31.086605165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=278af8730b02432a immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-xrqf\"}"
level=error ts=2026-05-05T00:22:31.089961603Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/278af8730b02432a/19df53734c7%3A19df54c2041%3A37ce0b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xrqf\"}"
level=debug ts=2026-05-05T00:22:31.958387808Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T00:22:35.291470548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:22:35.291965617Z caller=checkpoint.go:372 msg="writing series" size="656 kB"
level=debug ts=2026-05-05T00:22:35.291981806Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=8
level=info ts=2026-05-05T00:22:35.464382102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:22:35.467905806Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:22:35.680308684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:22:35.687958768Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:22:36.541574324Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:22:38.151110916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:22:38.154283596Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-05T00:22:38.624361235Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-05T00:22:38.753815495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:22:38.756980787Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:22:39.098106749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:22:39.101623141Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T00:22:40.291000666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:22:41.958587587Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-05T00:22:44.552746458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:22:44.561575523Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:22:44.960727561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:22:44.964634071Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:22:45.036738054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:22:45.040850796Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T00:22:45.272557703Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:22:45.272765335Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-05-05T00:22:45.291138364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:22:45.291176906Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:22:45.29118918Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:22:45.291196339Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:22:45.291696989Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
ts=2026-05-05T00:22:45.443428784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48156"
level=info ts=2026-05-05T00:22:47.711022756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:22:47.714963126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:22:47.715525764Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:22:47.718412167Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T00:22:48.429445914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:22:48.624518971Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-05T00:22:49.854595166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:22:49.858318585Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T00:22:50.290969431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:22:51.958642012Z caller=checkpoint.go:372 msg="writing series" size="476 kB"
level=debug ts=2026-05-05T00:22:51.958673329Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-05T00:22:53.353829735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2cb3d97231fbb42c immediate=false num_chunks=1 total_comp="601 kB" avg_comp="601 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lrta\"}"
level=error ts=2026-05-05T00:22:53.358218701Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2cb3d97231fbb42c/19df538ea33%3A19df54aa544%3Ab0e5845b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lrta\"}"
level=info ts=2026-05-05T00:22:54.874925811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:22:54.879482346Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=warn ts=2026-05-05T00:22:55.291466007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:22:55.291487232Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-05T00:22:55.600069977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:22:56.891817719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:22:56.895659137Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-05T00:22:58.625370979Z caller=checkpoint.go:372 msg="writing series" size="537 kB"
level=info ts=2026-05-05T00:22:58.874994276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:22:58.878315528Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:23:00.291245682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:23:01.180138487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:23:01.183555401Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:23:01.715170883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:23:01.718918425Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-05T00:23:01.958678299Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-05T00:23:02.791128605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:23:02.794590977Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T00:23:05.291120824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:23:05.291304438Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=debug ts=2026-05-05T00:23:05.29132265Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-05T00:23:08.624280599Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-05T00:23:10.29095536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:23:10.757906786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:23:10.762321107Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T00:23:11.661976148Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T00:23:11.958608627Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-05T00:23:11.95863159Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-05-05T00:23:12.098395032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:23:12.103411126Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:23:15.290638415Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:23:15.290664738Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:23:15.290673985Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:23:15.290674674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:23:15.291803145Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T00:23:15.387194713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:23:15.395141577Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:23:15.797035103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:23:15.800535334Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:23:16.224934771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=278af8730b02432a immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-xrqf\"}"
level=error ts=2026-05-05T00:23:16.228456634Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/278af8730b02432a/19df53734c7%3A19df54c2041%3A37ce0b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xrqf\"}"
level=info ts=2026-05-05T00:23:18.429732405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:23:18.432990692Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:23:18.433017791Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:23:18.433039832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:23:18.4356429Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-05T00:23:18.624433188Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
ts=2026-05-05T00:23:19.593815284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:47888"
level=info ts=2026-05-05T00:23:19.676654263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=09cac0c115a46ea5 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"build-e2e-hswr\"}"
level=error ts=2026-05-05T00:23:19.680419683Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9cac0c115a46ea5/19df53855d8%3A19df546a201%3A6c842155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hswr\"}"
level=warn ts=2026-05-05T00:23:20.291303004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:23:21.358399562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:23:21.36252095Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:23:21.958426539Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-05T00:23:25.023631065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:23:25.027420347Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T00:23:25.291411391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:23:25.291427841Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-05T00:23:25.502488871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:23:25.506370405Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
ts=2026-05-05T00:23:25.601879038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:23:25.822745084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:23:25.829963897Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:23:27.1104148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:23:27.118156199Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:23:28.624699032Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=info ts=2026-05-05T00:23:28.952393641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:23:28.956593172Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:23:30.108010878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:23:30.111283281Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-05T00:23:30.291433149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:23:31.671200009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:23:31.675030926Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T00:23:31.708110632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="804 B" avg_comp="804 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:23:31.711437884Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:23:31.711456891Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:23:31.711479122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:23:31.714452061Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:23:31.961103381Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-05T00:23:31.961126119Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=6
level=info ts=2026-05-05T00:23:32.652339018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:23:32.655981612Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T00:23:33.227490755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:23:33.23066644Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:23:33.230686847Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:23:33.230707377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:23:33.23337202Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T00:23:34.537030254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T00:23:35.29112713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:23:35.291202767Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-05T00:23:35.930672488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:23:35.933893153Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:23:37.102230704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:23:37.110571538Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=debug ts=2026-05-05T00:23:38.625168692Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=debug ts=2026-05-05T00:23:39.218006958Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:23:39.412638968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:23:39.415741858Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:23:39.41576202Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:23:39.415783458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:23:39.418432383Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:23:40.290952995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:23:41.633094735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:23:41.637322946Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:23:41.637343641Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:23:41.637365857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:23:41.641722459Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-05T00:23:41.95785187Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-05T00:23:43.260380112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:23:43.26350847Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:23:44.209329633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:23:44.212995099Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:23:45.272683399Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:23:45.273052189Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-05T00:23:45.291421966Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-05T00:23:45.291430558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:23:45.29145503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:23:45.291468749Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:23:45.291477623Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T00:23:45.411135983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2cb3d97231fbb42c immediate=false num_chunks=1 total_comp="601 kB" avg_comp="601 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lrta\"}"
level=error ts=2026-05-05T00:23:45.417314836Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2cb3d97231fbb42c/19df538ea33%3A19df54aa544%3Ab0e5845b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lrta\"}"
level=info ts=2026-05-05T00:23:47.229763867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:23:47.233184705Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T00:23:48.625290394Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-05T00:23:48.799818749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:23:48.803145868Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T00:23:50.291610363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:23:51.216670995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:23:51.221506753Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=debug ts=2026-05-05T00:23:51.959067512Z caller=checkpoint.go:372 msg="writing series" size="534 kB"
level=debug ts=2026-05-05T00:23:51.95909095Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-05T00:23:52.031849732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:23:52.035517618Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:23:53.118675213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:23:53.12321402Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T00:23:54.997118323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51606"
level=warn ts=2026-05-05T00:23:55.290643074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:23:55.291993826Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
ts=2026-05-05T00:23:55.603813789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=info ts=2026-05-05T00:23:57.850388272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:23:57.853622385Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:23:58.625028843Z caller=checkpoint.go:372 msg="writing series" size="562 kB"
level=info ts=2026-05-05T00:23:59.125221097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:23:59.128749182Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-05T00:24:00.291451513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:24:01.957875885Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-05T00:24:04.69425228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:24:04.697775532Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:24:04.840031763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:24:04.843759112Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:24:05.025313151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=09cac0c115a46ea5 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"build-e2e-hswr\"}"
level=error ts=2026-05-05T00:24:05.029174985Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9cac0c115a46ea5/19df53855d8%3A19df546a201%3A6c842155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hswr\"}"
level=warn ts=2026-05-05T00:24:05.290801367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:24:05.290887758Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-05T00:24:07.102391352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:24:07.108200893Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T00:24:08.627149754Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T00:24:08.627172206Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=5
level=info ts=2026-05-05T00:24:09.139817435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=278af8730b02432a immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-xrqf\"}"
level=error ts=2026-05-05T00:24:09.143027866Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/278af8730b02432a/19df53734c7%3A19df54c2041%3A37ce0b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xrqf\"}"
level=warn ts=2026-05-05T00:24:10.291348704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:24:10.664996023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T00:24:11.49209787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:24:11.499242986Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T00:24:11.958511221Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=debug ts=2026-05-05T00:24:13.073910729Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:24:13.175063958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:24:13.178413931Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:24:13.32292459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:24:13.325928407Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:24:14.026390069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:24:14.029531177Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:24:14.735637824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:24:14.739652794Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T00:24:15.184128028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:24:15.1873794Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:24:15.291368969Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T00:24:15.291382669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:24:15.291392905Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:24:15.291402559Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:24:15.291400914Z caller=checkpoint.go:384 msg="flushing series" totalSize="245 kB" series=1
level=info ts=2026-05-05T00:24:15.299251704Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-05T00:24:15.300752349Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009939174s
ts=2026-05-05T00:24:15.447375606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59488"
level=info ts=2026-05-05T00:24:20.055672571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:24:20.062642861Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-05T00:24:20.291117845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:24:22.554970105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:24:22.562903437Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:24:22.902977322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:24:22.906547193Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:24:23.226396194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:24:23.230097059Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:24:23.230116707Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:24:23.23014065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:24:23.233326943Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:24:23.639632054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:24:23.643645581Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T00:24:24.547316049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:24:24.55056062Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T00:24:25.290900089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T00:24:25.605324938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:24:27.677316526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2cb3d97231fbb42c immediate=false num_chunks=1 total_comp="601 kB" avg_comp="601 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lrta\"}"
level=error ts=2026-05-05T00:24:27.681730791Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2cb3d97231fbb42c/19df538ea33%3A19df54aa544%3Ab0e5845b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lrta\"}"
level=info ts=2026-05-05T00:24:28.28443856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:24:28.287645033Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:24:28.287669042Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:24:28.287693566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:24:28.290526549Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:24:28.558256077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:24:28.561684086Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:24:30.290767496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:24:30.657548382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:24:30.66090342Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T00:24:31.380312876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:24:31.384249173Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:24:33.038300245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:24:33.043010115Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=info ts=2026-05-05T00:24:33.261524337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:24:33.264751991Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:24:33.264771332Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:24:33.264801463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:24:33.267525673Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:24:34.707714528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:24:34.713875597Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T00:24:35.291309168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:24:39.867096543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:24:39.871151105Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T00:24:40.291095483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:24:45.272588626Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:24:45.27299831Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:24:45.291361714Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-05T00:24:45.29139602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:24:45.291398515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:24:45.291418308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:24:45.291427194Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T00:24:45.291608995Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-05T00:24:45.291621041Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T00:24:45.291642514Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.109µs success=true
level=debug ts=2026-05-05T00:24:45.291846601Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-05T00:24:46.354171592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:24:46.357771082Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T00:24:46.635516697Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:24:46.677782326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:24:46.681959513Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-05T00:24:48.558506079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:24:48.561630836Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:24:48.62592805Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-05T00:24:50.119089678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=278af8730b02432a immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-xrqf\"}"
level=error ts=2026-05-05T00:24:50.122582614Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/278af8730b02432a/19df53734c7%3A19df54c2041%3A37ce0b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xrqf\"}"
level=warn ts=2026-05-05T00:24:50.291090214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:24:51.900760334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:24:51.907392195Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T00:24:51.959547098Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
ts=2026-05-05T00:24:55.000357386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36990"
level=info ts=2026-05-05T00:24:55.206764083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:24:55.210115376Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T00:24:55.291537738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:24:55.292621594Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-05-05T00:24:55.606878454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:24:55.798530206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:24:55.803456033Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T00:24:56.753832419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:24:56.757147497Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:24:56.757170291Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:24:56.757191936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:24:56.75990941Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:24:57.009724764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=09cac0c115a46ea5 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"build-e2e-hswr\"}"
level=error ts=2026-05-05T00:24:57.013681945Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9cac0c115a46ea5/19df53855d8%3A19df546a201%3A6c842155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hswr\"}"
level=debug ts=2026-05-05T00:24:58.625551049Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=warn ts=2026-05-05T00:25:00.291511241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:25:01.959298932Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T00:25:02.30355215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:25:02.307062096Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:25:03.143318994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:25:03.146661875Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:25:03.146681833Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:25:03.146703279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:25:03.149526402Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T00:25:03.831567975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:25:03.834939426Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:25:04.057483991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:25:04.064112705Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T00:25:05.290576006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:25:05.292706913Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-05T00:25:05.50044485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:25:05.50431397Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:25:08.636507021Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-05T00:25:08.636530046Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=info ts=2026-05-05T00:25:09.357647437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:25:09.361131146Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T00:25:10.182229236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T00:25:10.290571504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:25:10.858053982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:25:10.861802311Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T00:25:11.403588307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:25:11.406860168Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T00:25:11.863891286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:25:11.867054828Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:25:11.867077318Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:25:11.867097848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:25:11.869830443Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-05T00:25:11.970417482Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-05T00:25:11.970438703Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=1
level=info ts=2026-05-05T00:25:14.209564173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2cb3d97231fbb42c immediate=false num_chunks=1 total_comp="601 kB" avg_comp="601 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lrta\"}"
level=error ts=2026-05-05T00:25:14.213778412Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2cb3d97231fbb42c/19df538ea33%3A19df54aa544%3Ab0e5845b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lrta\"}"
level=info ts=2026-05-05T00:25:14.631037196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:25:14.63780743Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T00:25:15.290696362Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T00:25:15.290705385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:25:15.290725157Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:25:15.290750001Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:25:15.291941672Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-05-05T00:25:15.450466466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45018"
level=info ts=2026-05-05T00:25:15.46786583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:25:15.471003779Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:25:15.481051063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:25:15.48532726Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=info ts=2026-05-05T00:25:15.914493442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:25:15.918829593Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T00:25:16.436319762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:25:16.443407724Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=info ts=2026-05-05T00:25:17.053024624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:25:17.05613414Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T00:25:17.751314433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:25:17.755242468Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-05T00:25:18.625988027Z caller=checkpoint.go:372 msg="writing series" size="484 kB"
level=warn ts=2026-05-05T00:25:20.291614738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:25:21.692367195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:25:21.695788196Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:25:21.959467532Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=debug ts=2026-05-05T00:25:22.210444758Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:25:23.100140528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:25:23.103329318Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-05T00:25:24.16640276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:25:24.170495099Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:25:24.170517333Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:25:24.170540283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:25:24.173952256Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:25:24.439748683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:25:24.442932636Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-05T00:25:25.001588169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53754"
level=warn ts=2026-05-05T00:25:25.290801981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:25:25.291996142Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-05T00:25:25.608846509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=info ts=2026-05-05T00:25:28.171317581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:25:28.174877118Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T00:25:28.630678351Z caller=checkpoint.go:372 msg="writing series" size="4.9 MB"
level=debug ts=2026-05-05T00:25:28.630699309Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=5
level=warn ts=2026-05-05T00:25:30.291996911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:25:31.959516254Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=warn ts=2026-05-05T00:25:35.291451324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:25:35.292622021Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=info ts=2026-05-05T00:25:37.406888662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:25:37.411674296Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:25:37.460724058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=278af8730b02432a immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-xrqf\"}"
level=error ts=2026-05-05T00:25:37.463584821Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/278af8730b02432a/19df53734c7%3A19df54c2041%3A37ce0b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xrqf\"}"
level=info ts=2026-05-05T00:25:38.352312105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-05T00:25:38.356171294Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19df52edf14%3A19df5783553%3Affe0161e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-05T00:25:38.625958276Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-05T00:25:40.291528561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:25:40.56465327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:25:40.571674512Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:25:40.571701538Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:25:40.571736538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:25:40.578427886Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T00:25:42.019133501Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-05T00:25:42.019168872Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=4
level=info ts=2026-05-05T00:25:44.831283612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=09cac0c115a46ea5 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"build-e2e-hswr\"}"
level=error ts=2026-05-05T00:25:44.841359349Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9cac0c115a46ea5/19df53855d8%3A19df546a201%3A6c842155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hswr\"}"
level=info ts=2026-05-05T00:25:45.272529182Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T00:25:45.273374031Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-05-05T00:25:45.290788281Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:25:45.290821248Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:25:45.290832261Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T00:25:45.290953273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:25:45.292085141Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-05T00:25:46.069306409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T00:25:46.079315659Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df52f06f4%3A19df5428bd5%3A89add556?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T00:25:47.377103043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="822 B" avg_comp="822 B" total_uncomp="4.4 kB" avg_uncomp="4.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T00:25:47.392574266Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df52f06f4%3A19df52f9702%3A5617586b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T00:25:48.415676045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T00:25:48.426483646Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df537c54f%3A19df539152b%3A9ee424a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T00:25:48.627113407Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
ts=2026-05-05T00:25:49.603728119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59384"
level=warn ts=2026-05-05T00:25:50.290921276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:25:50.595927901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cad5843181b1923 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-acuq\"}"
level=error ts=2026-05-05T00:25:50.607020314Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cad5843181b1923/19df5372ce3%3A19df5375409%3Ae88305f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-acuq\"}"
level=info ts=2026-05-05T00:25:50.689450269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="123 kB" avg_uncomp="123 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T00:25:50.699525716Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df52ede64%3A19df530b88b%3A7f721d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T00:25:51.958981145Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-05T00:25:52.153364506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T00:25:52.164234546Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df52edff7%3A19df571a285%3Ab655fbf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T00:25:55.291068571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:25:55.292133704Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
ts=2026-05-05T00:25:55.610422548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=info ts=2026-05-05T00:25:55.644767603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-05T00:25:55.648108361Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df52f575e%3A19df5307a20%3A9b4ba8aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-05T00:25:56.979663272Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T00:25:58.019154097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T00:25:58.02332798Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df52ed77c%3A19df5461561%3A7ea0bd8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T00:25:58.625880936Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-05T00:25:59.85006665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T00:25:59.857146102Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df52ee133%3A19df545bd52%3A11c8068d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T00:25:59.96049134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T00:25:59.967896894Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df52f06f4%3A19df551c194%3Afe550c10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T00:26:00.053150496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:26:00.059931997Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:26:00.059952818Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:26:00.059978583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:26:00.066062722Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-05T00:26:00.290631884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:26:01.959076699Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-05T00:26:01.959099881Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-05-05T00:26:02.990895399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:26:02.992512477Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T00:26:03.010586076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=82068ba8f500899d immediate=false num_chunks=1 total_comp="532 kB" avg_comp="532 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"build-e2e-hlyn\"}"
level=error ts=2026-05-05T00:26:03.014317533Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/82068ba8f500899d/19df53730cb%3A19df544ec6c%3Ab53be1fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hlyn\"}"
level=info ts=2026-05-05T00:26:03.222891153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2cb3d97231fbb42c immediate=false num_chunks=1 total_comp="601 kB" avg_comp="601 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lrta\"}"
level=error ts=2026-05-05T00:26:03.227515159Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2cb3d97231fbb42c/19df538ea33%3A19df54aa544%3Ab0e5845b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lrta\"}"
level=info ts=2026-05-05T00:26:03.496456481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="218 kB" avg_uncomp="218 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-05T00:26:03.499787155Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19df52ed8c3%3A19df52fee35%3Af82e57bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-05T00:26:03.751310118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T00:26:03.754378725Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df52f06f4%3A19df52f9702%3A7a16309e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T00:26:05.290953473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T00:26:05.292471121Z caller=checkpoint.go:372 msg="writing series" size="572 kB"
level=info ts=2026-05-05T00:26:06.032002358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:26:06.036379623Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:26:06.036404915Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:26:06.036427765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:26:06.040007108Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T00:26:06.317069876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-05T00:26:06.321055745Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df52ed775%3A19df577e415%3Aed756fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-05T00:26:07.327907086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T00:26:07.332983271Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df52ed7f3%3A19df5710645%3Ace5f359c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T00:26:08.545433677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T00:26:08.54866343Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df52f06f4%3A19df52f57bf%3A72d73598?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T00:26:08.625452425Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-05T00:26:09.018331361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:26:09.022005963Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:26:09.02202898Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T00:26:09.022064572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T00:26:09.025315693Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df52edea0%3A19df531e180%3A683bb526?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T00:26:10.291108311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:26:11.478454592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T00:26:11.486352878Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df52ed9fc%3A19df52f4fba%3Ae89752e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T00:26:11.959591337Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=info ts=2026-05-05T00:26:13.334154649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=776db9291a3e901e immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-wzoq\"}"
level=error ts=2026-05-05T00:26:13.340666127Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/776db9291a3e901e/19df537a5fb%3A19df55ca2d1%3A619b35ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-wzoq\"}"
level=warn ts=2026-05-05T00:26:15.290848973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T00:26:15.29087033Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T00:26:15.29088774Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T00:26:15.290897907Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T00:26:15.292094887Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-05T00:26:15.453115995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44192"
level=info ts=2026-05-05T00:26:17.082998119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T00:26:17.086536096Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df52f4fba%3A19df52f9701%3A5424be9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T00:26:17.804588608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T00:26:17.810275086Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df52ede60%3A19df57ed0bc%3A20d1a637?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T00:26:18.625526725Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
ts=2026-05-05T00:26:19.606193238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40180"
level=info ts=2026-05-05T00:26:19.749217611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T00:26:19.753234212Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df52ede5c%3A19df52f06f5%3A4c05a3cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T00:26:20.290884809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"