level=info ts=2026-04-24T23:43:31.863903976Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)" level=info ts=2026-04-24T23:43:31.863929416Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml level=debug ts=2026-04-24T23:43:31.864202923Z caller=modules.go:1984 msg="initializing ingester query tags interceptors" level=info ts=2026-04-24T23:43:31.864799659Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095 level=debug ts=2026-04-24T23:43:31.865143222Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-24T23:43:31.865206486Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.60 inf=eth0 level=info ts=2026-04-24T23:43:31.865703526Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-f6da1526 ts=2026-04-24T23:43:31.865778716Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired" level=debug ts=2026-04-24T23:43:31.865796623Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.60 advertisePort=7946 level=debug ts=2026-04-24T23:43:31.865821557Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.60 advertisePort=7946 level=info ts=2026-04-24T23:43:31.865958051Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4 level=debug ts=2026-04-24T23:43:31.867406992Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:43:31.86824799Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode" level=info ts=2026-04-24T23:43:31.868395163Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-24T23:43:31.869203367Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0 level=debug ts=2026-04-24T23:43:31.869220034Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-24T23:43:31.869219101Z level=debug ts=2026-04-24T23:43:31.86923163Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads" level=info ts=2026-04-24T23:43:31.869243719Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0 level=info ts=2026-04-24T23:43:31.869256289Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1 level=debug ts=2026-04-24T23:43:31.869260669Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-04-24T23:43:31.86926024Z level=debug ts=2026-04-24T23:43:31.869264121Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads" level=debug ts=2026-04-24T23:43:31.87036839Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-24T23:43:31.870428652Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.60 inf=eth0 ts=2026-04-24T23:43:31.87428362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.92:7946" ts=2026-04-24T23:43:31.876596175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.91:7946" ts=2026-04-24T23:43:31.878983777Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.90:7946: dial tcp 10.128.0.90:7946: connect: connection refused" ts=2026-04-24T23:43:31.881317116Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.83:7946: dial tcp 10.129.0.83:7946: connect: connection refused" ts=2026-04-24T23:43:31.881874527Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.57:7946: dial tcp 10.130.0.57:7946: connect: connection refused" ts=2026-04-24T23:43:31.882016422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.130.0.59:7946" level=info ts=2026-04-24T23:43:31.882573529Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=16.6178ms level=info ts=2026-04-24T23:43:31.88259944Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local level=debug ts=2026-04-24T23:43:31.88311091Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config level=debug ts=2026-04-24T23:43:31.883122055Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server level=info ts=2026-04-24T23:43:31.883117521Z caller=module_service.go:82 msg=starting module=runtime-config level=debug ts=2026-04-24T23:43:31.883134916Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv level=debug ts=2026-04-24T23:43:31.883143207Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server level=debug ts=2026-04-24T23:43:31.8831447Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv level=info ts=2026-04-24T23:43:31.883198248Z caller=module_service.go:82 msg=starting module=server level=debug ts=2026-04-24T23:43:31.883479753Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server level=debug ts=2026-04-24T23:43:31.883489607Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store level=debug ts=2026-04-24T23:43:31.883505876Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv level=info ts=2026-04-24T23:43:31.883566984Z caller=module_service.go:82 msg=starting module=memberlist-kv level=debug ts=2026-04-24T23:43:31.883726923Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config level=info ts=2026-04-24T23:43:31.883751603Z caller=module_service.go:82 msg=starting module=store level=debug ts=2026-04-24T23:43:31.883758738Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring level=debug ts=2026-04-24T23:43:31.883762618Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config level=debug ts=2026-04-24T23:43:31.883765787Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server level=info ts=2026-04-24T23:43:31.883768586Z caller=module_service.go:82 msg=starting module=ring level=debug ts=2026-04-24T23:43:31.883973013Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config level=debug ts=2026-04-24T23:43:31.883977134Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server level=info ts=2026-04-24T23:43:31.883980285Z caller=module_service.go:82 msg=starting module=analytics level=debug ts=2026-04-24T23:43:31.88404523Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics level=debug ts=2026-04-24T23:43:31.884061696Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv level=debug ts=2026-04-24T23:43:31.884064397Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring level=info ts=2026-04-24T23:43:31.884067058Z caller=module_service.go:82 msg=starting module=ingester level=info ts=2026-04-24T23:43:31.88408772Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint" ts=2026-04-24T23:43:31.885907924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.92:7946" ts=2026-04-24T23:43:31.888170935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.91:7946" ts=2026-04-24T23:43:31.890294132Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.90:7946: dial tcp 10.128.0.90:7946: connect: connection refused" ts=2026-04-24T23:43:31.891416226Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.83:7946: dial tcp 10.129.0.83:7946: connect: connection refused" ts=2026-04-24T23:43:31.891521986Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.57:7946: dial tcp 10.130.0.57:7946: connect: connection refused" ts=2026-04-24T23:43:31.891642913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.130.0.59:7946" level=info ts=2026-04-24T23:43:31.892119652Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=9.515011ms level=info ts=2026-04-24T23:43:32.025597816Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=141.511434ms errors=false level=info ts=2026-04-24T23:43:32.02562398Z caller=ingester.go:587 component=ingester msg="recovering from WAL" level=info ts=2026-04-24T23:43:32.072218323Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=188.13198ms errors=false level=info ts=2026-04-24T23:43:32.072242438Z caller=ingester.go:551 component=ingester msg="closing recoverer" level=info ts=2026-04-24T23:43:32.072267643Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=188.181882ms level=info ts=2026-04-24T23:43:32.072290076Z caller=wal.go:158 msg=started component=wal level=info ts=2026-04-24T23:43:32.072318158Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=17.569073442s level=info ts=2026-04-24T23:43:32.072348184Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty" level=info ts=2026-04-24T23:43:32.072359278Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat" level=info ts=2026-04-24T23:43:32.072365772Z caller=loki.go:599 msg="Loki started" startup_time=261.522418ms level=info ts=2026-04-24T23:43:32.072391807Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z level=debug ts=2026-04-24T23:43:32.072420581Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester ts=2026-04-24T23:43:32.165623242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:33164" ts=2026-04-24T23:43:33.708688818Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.130.0.60:7946" ts=2026-04-24T23:43:33.711400757Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.92:7946" ts=2026-04-24T23:43:33.711458204Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.91:7946" ts=2026-04-24T23:43:34.160518611Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.91:7946" ts=2026-04-24T23:43:36.161660227Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.92:7946" ts=2026-04-24T23:43:36.161694385Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.130.0.59:7946" ts=2026-04-24T23:43:36.161703827Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.130.0.60:7946" level=warn ts=2026-04-24T23:43:37.07347858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:43:42.073379834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:43:47.073437925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:43:51.866975225Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.130.0.60:7946" level=warn ts=2026-04-24T23:43:52.072612141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:43:53.867368347Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c2f45ad5 (timeout reached)" ts=2026-04-24T23:43:53.86809967Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.130.0.59:7946" ts=2026-04-24T23:43:53.870747557Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.92:7946" ts=2026-04-24T23:43:53.870833546Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.91:7946" ts=2026-04-24T23:43:54.160812231Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.91:7946" level=debug ts=2026-04-24T23:43:55.320838815Z caller=logging.go:137 msg="GET /ready (503) 103.04µs" ts=2026-04-24T23:43:56.162084123Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.92:7946" ts=2026-04-24T23:43:56.162124947Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.130.0.59:7946" ts=2026-04-24T23:43:56.162169459Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.130.0.60:7946" ts=2026-04-24T23:43:56.707770666Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-c2f45ad5 as failed, suspect timeout reached (2 peer confirmations)" ts=2026-04-24T23:43:56.707800282Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.130.0.59:7946" ts=2026-04-24T23:43:56.867417955Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c2f45ad5 has failed, no acks received" level=warn ts=2026-04-24T23:43:57.07275511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:43:58.709120776Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.130.0.60:7946" ts=2026-04-24T23:43:58.711458214Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.92:7946" ts=2026-04-24T23:43:58.711486753Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-c2f45ad5' from=10.128.0.91:7946" level=info ts=2026-04-24T23:44:02.073113911Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=info ts=2026-04-24T23:44:02.073156583Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership" level=info ts=2026-04-24T23:44:02.07316883Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000 level=info ts=2026-04-24T23:44:02.073190943Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:44:02.073202152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:44:02.168685707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39458" level=debug ts=2026-04-24T23:44:05.319561633Z caller=logging.go:137 msg="GET /ready (503) 66.498µs" level=warn ts=2026-04-24T23:44:07.073204542Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:44:12.072916504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:44:17.073191216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:44:22.073046422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:44:24.378252865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-24T23:44:24.750405136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=info ts=2026-04-24T23:44:25.02906398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="132 B" avg_comp="132 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:44:25.029121914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=debug ts=2026-04-24T23:44:25.029372666Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:44:25.029438773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:44:25.036420128Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:44:25.036429898Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:44:25.036435509Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:44:25.329634816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:44:25.336583246Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:44:25.630468851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:44:25.634463972Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:44:25.930466191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:44:25.934126912Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:44:26.229939159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:44:26.233458663Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:44:26.530342621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:44:26.537114369Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:44:26.830274918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:44:26.836795709Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-24T23:44:27.073359369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:44:27.130525368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:44:27.134926662Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-24T23:44:27.43042104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="344 B" avg_uncomp="344 B" idle=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:44:27.433936597Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-24T23:44:27.729535227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:44:27.732909414Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:44:28.030156396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="154 kB" avg_uncomp="154 kB" idle=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:44:28.033678351Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:44:28.329687234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:44:28.336577205Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-24T23:44:28.629827551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:44:28.63330992Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:44:28.930359915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:44:28.93380282Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:44:29.230461822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:44:29.233965097Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:44:29.529726354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:44:29.533072852Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:44:31.868614433Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:44:31.868937926Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:44:32.073015599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-24T23:44:32.073039366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:44:32.073052549Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:44:32.073063665Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:44:37.072890246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:44:38.479158867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:44:38.48601033Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:44:39.205942455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:44:39.213681497Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-24T23:44:39.963566122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:44:39.967028067Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:44:40.094427541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:44:40.097880386Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:44:40.682993016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:44:40.686483007Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:44:40.70353838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:44:40.706800188Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:44:41.179053031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:44:41.18247168Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:44:41.719521111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:44:41.723417045Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" ts=2026-04-24T23:44:42.007751633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55812" level=warn ts=2026-04-24T23:44:42.073438886Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:44:42.985788751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:44:42.992623068Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:44:43.269498583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:44:43.272849933Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:44:43.518456551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:44:43.522235321Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:44:43.632438654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:44:43.638985404Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:44:43.91906341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:44:43.923901898Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:44:44.02307904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:44:44.026509713Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-24T23:44:44.174636607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:44:44.178165752Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:44:44.449459209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:44:44.454728635Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-24T23:44:45.824622324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-24T23:44:46.021364111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:44:46.024926289Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:44:46.340780197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:44:46.345125061Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=warn ts=2026-04-24T23:44:47.07262666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:44:52.072536775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:44:54.751820702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=warn ts=2026-04-24T23:44:57.073269677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:44:57.408978099Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:45:02.072613644Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-24T23:45:02.072632714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:45:02.072647529Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:45:02.072657008Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:45:02.770609898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-24T23:45:02.887147531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:45:02.891820957Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:45:03.946623786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:45:03.953142969Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:45:05.143749413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:45:05.150082004Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:45:05.86515203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:45:05.871665917Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-24T23:45:07.072906014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:45:07.477003978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:45:07.480240309Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:45:08.753459643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:45:08.756762771Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:45:10.579955556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:45:10.585856379Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:45:11.358836245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:45:11.362245345Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:45:11.436368946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:45:11.439626044Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:45:11.583242909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:45:11.586458051Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:45:11.722872102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:45:11.726311658Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" ts=2026-04-24T23:45:12.009348369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55610" level=warn ts=2026-04-24T23:45:12.073253912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:45:12.207530401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:45:12.210865787Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:45:12.59302638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:45:12.599205895Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-24T23:45:13.669426152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:45:13.672693925Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:45:15.904731294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:45:15.911485573Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:45:16.517979671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:45:16.522344362Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-24T23:45:16.941677613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:45:16.947499118Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-24T23:45:17.073096471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:45:17.911417592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:45:17.914980501Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-24T23:45:22.072655821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:45:24.753804189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-24T23:45:25.215771581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-04-24T23:45:26.677785161Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-24T23:45:27.073184045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:45:31.868624194Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:45:31.869064069Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=debug ts=2026-04-24T23:45:31.870444678Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads" level=debug ts=2026-04-24T23:45:31.870494337Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974526 n=1 level=debug ts=2026-04-24T23:45:31.870536032Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974526 n=1 level=info ts=2026-04-24T23:45:32.073169375Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-24T23:45:32.073168622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:45:32.073200556Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:45:32.073209089Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:45:37.07258991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:45:42.01080163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:50822" level=warn ts=2026-04-24T23:45:42.072679898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:45:47.073080562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:45:51.4911254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:45:51.498635424Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-24T23:45:52.072654595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:45:52.599823391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:45:52.603597542Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:45:53.064873501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:45:53.068377463Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-24T23:45:53.43005678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-04-24T23:45:53.431996436Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-04-24T23:45:53.584518788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:45:53.588416538Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:45:53.909506514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:45:53.912760924Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" ts=2026-04-24T23:45:54.75679949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-24T23:45:56.870531444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:45:56.87415564Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-24T23:45:57.044544695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-24T23:45:57.072905229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:45:58.356198213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:45:58.363233438Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:45:58.732344409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:45:58.737465071Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:45:58.91514169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:45:58.91861273Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:45:59.073158237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:45:59.076952457Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:45:59.111031975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:45:59.117832064Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-24T23:46:00.289041752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:46:00.292629419Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:46:01.034500939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:46:01.03825413Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:46:01.237825235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:46:01.244562278Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:46:02.073351012Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:46:02.07338816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:46:02.073397251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:46:02.073460551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:46:02.611558472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:46:02.616825287Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-24T23:46:05.367369961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:46:05.370707536Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:46:05.759684758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:46:05.762857663Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-24T23:46:06.470903413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:46:06.475256388Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=warn ts=2026-04-24T23:46:07.073103143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:46:12.073090391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:46:17.073318744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:46:22.072825031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:46:23.36814832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:52124" ts=2026-04-24T23:46:24.759049302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=warn ts=2026-04-24T23:46:27.072806271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:46:28.483977587Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:46:31.868612616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:46:31.868948084Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:46:32.073046081Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:46:32.07308065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:46:32.073091904Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:46:32.073106377Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:46:32.09099026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-24T23:46:32.17778094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:52220" level=warn ts=2026-04-24T23:46:37.072977014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:46:40.159599022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:46:40.167182789Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-24T23:46:42.073005995Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:46:42.938295825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:46:42.94211187Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:46:43.310126726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:46:43.313787867Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:46:43.342849743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:46:43.346529375Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:46:44.233917941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:46:44.240852314Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:46:44.439772955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:46:44.446705957Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:46:44.570066256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:46:44.574734668Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:46:45.033177615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:46:45.036755558Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:46:45.522370946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:46:45.52596294Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:46:45.88317917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:46:45.886594505Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=warn ts=2026-04-24T23:46:47.073226323Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:46:49.433611893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:46:49.437704136Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:46:51.791457283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:46:51.798097119Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:46:51.994670097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:46:51.998289599Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-24T23:46:52.072793322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:46:52.34569851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=info ts=2026-04-24T23:46:52.35039997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:46:52.35159838Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:46:52.353257866Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" ts=2026-04-24T23:46:53.370637974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:42330" ts=2026-04-24T23:46:54.761147224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=info ts=2026-04-24T23:46:54.985517524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:46:54.9888052Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-24T23:46:57.073075403Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:46:59.424143945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:46:59.427586471Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-24T23:47:02.073443071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:47:02.073436684Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:47:02.073469352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:47:02.073478969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-24T23:47:02.418830722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:47:02.423780091Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=debug ts=2026-04-24T23:47:02.663665211Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-24T23:47:07.072745877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:47:12.073075323Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:47:17.073433966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:47:22.073313725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:47:22.850122914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:47:22.858503941Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" ts=2026-04-24T23:47:24.762529447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=info ts=2026-04-24T23:47:25.555248329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:47:25.558665919Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:47:26.146007218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:47:26.149968466Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-24T23:47:27.073151737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:47:27.767929029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:47:27.772680144Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:47:28.617102672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:47:28.623139327Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:47:29.99144863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:47:29.999439863Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-24T23:47:31.868616199Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:47:31.868675654Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-24T23:47:32.073130498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:47:32.073167385Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:47:32.073178394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:47:32.073230018Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:47:32.083337534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:47:32.090369074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-24T23:47:32.499694842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-24T23:47:32.581510498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:47:32.587972933Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-24T23:47:33.770464954Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:47:35.199100891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:47:35.203107773Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:47:35.750549079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:47:35.753902538Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:47:35.888508786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:47:35.892447247Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=warn ts=2026-04-24T23:47:37.073484323Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:47:38.464366004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:47:38.467829636Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-24T23:47:42.07313933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:47:42.553893158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:47:42.558487352Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-24T23:47:44.678915059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:47:44.68219036Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:47:44.750378507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:47:44.755584915Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-24T23:47:44.947294566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:47:44.95096649Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:47:46.955769977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:47:46.95944336Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-24T23:47:47.073063562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:47:52.073081177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:47:54.764346546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-24T23:47:54.910174594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:47:54.914439531Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-24T23:47:57.073098116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:48:02.073275511Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:48:02.073310114Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:48:02.073319586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:48:02.07335205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-24T23:48:07.072943779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:48:08.336169148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:48:08.343113302Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-24T23:48:08.931986181Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:48:09.297910346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:48:09.305581546Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-24T23:48:12.072974645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:48:12.667098419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:48:12.672635211Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:48:16.728587532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:48:16.732068932Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-24T23:48:17.073500694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:48:17.865588015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:48:17.869559746Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:48:21.882805418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:48:21.888210763Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-24T23:48:22.072518967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:48:23.900971937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:48:23.904599872Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" ts=2026-04-24T23:48:24.767736457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-24T23:48:26.035193302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:48:26.043596263Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-24T23:48:27.072993591Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:48:27.44845524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:48:27.45342331Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-24T23:48:27.570993747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:48:27.578150418Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:48:27.876202662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:48:27.879719066Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:48:28.775890675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:48:28.782968442Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:48:31.868654398Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:48:31.868885638Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:48:32.072509437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=info ts=2026-04-24T23:48:32.072512401Z caller=checkpoint.go:612 msg="starting checkpoint" level=warn ts=2026-04-24T23:48:32.07252179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:48:32.072535201Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:48:32.072543763Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-24T23:48:32.072659481Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014 level=debug ts=2026-04-24T23:48:32.074376201Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB" level=debug ts=2026-04-24T23:48:32.074387409Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=1 level=info ts=2026-04-24T23:48:32.28727527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:48:32.29072804Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:48:32.452738461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:48:32.45727493Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-24T23:48:32.930703818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:48:32.933718987Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-24T23:48:33.775405243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:48:34.960552525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:48:34.963956232Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=debug ts=2026-04-24T23:48:35.406981926Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB" level=info ts=2026-04-24T23:48:35.993345531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:48:35.996358084Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-24T23:48:37.072948503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:48:38.740084824Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB" ts=2026-04-24T23:48:42.028243531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:54730" level=warn ts=2026-04-24T23:48:42.072998521Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:48:42.073052191Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=info ts=2026-04-24T23:48:42.731755652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:48:42.735612936Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-24T23:48:45.407037567Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=warn ts=2026-04-24T23:48:47.07325862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:48:48.740159082Z caller=checkpoint.go:378 msg="writing series" size="134 kB" level=debug ts=2026-04-24T23:48:49.953034416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-24T23:48:50.122602186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:48:50.130861048Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-24T23:48:52.072582216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:48:52.073696362Z caller=checkpoint.go:378 msg="writing series" size="13 kB" ts=2026-04-24T23:48:54.770892048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=debug ts=2026-04-24T23:48:55.406715904Z caller=checkpoint.go:378 msg="writing series" size="25 kB" level=warn ts=2026-04-24T23:48:57.072828784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:48:58.713170691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:48:58.735086529Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-24T23:48:58.740145267Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB" level=warn ts=2026-04-24T23:49:02.073186454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:49:02.073203579Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:49:02.073227577Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:49:02.073236931Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:49:02.073492252Z caller=checkpoint.go:378 msg="writing series" size="92 kB" ts=2026-04-24T23:49:02.18726915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:57438" level=debug ts=2026-04-24T23:49:03.070228859Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-24T23:49:05.408301241Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB" level=debug ts=2026-04-24T23:49:05.408342594Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=10 level=warn ts=2026-04-24T23:49:07.073371818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:49:07.229372546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:49:07.233025163Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:49:07.667550317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:49:07.671232895Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-24T23:49:08.740241967Z caller=checkpoint.go:378 msg="writing series" size="165 kB" level=info ts=2026-04-24T23:49:10.252478123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:49:10.258069496Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:49:10.819587749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:49:10.823082458Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:49:11.240307199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:49:11.249066765Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" ts=2026-04-24T23:49:12.030061713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:41362" level=warn ts=2026-04-24T23:49:12.073176319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:49:12.07320579Z caller=checkpoint.go:378 msg="writing series" size="97 kB" level=info ts=2026-04-24T23:49:13.190346755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:49:13.197642583Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:49:13.820278243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:49:13.823880916Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:49:14.195117895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:49:14.201760172Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:49:15.099245467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:49:15.103049614Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-24T23:49:15.406734643Z caller=checkpoint.go:378 msg="writing series" size="41 kB" level=info ts=2026-04-24T23:49:15.552116496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:49:15.557030704Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-24T23:49:16.611663557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:49:16.615022939Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=warn ts=2026-04-24T23:49:17.072739196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:49:18.740219616Z caller=checkpoint.go:378 msg="writing series" size="389 kB" level=info ts=2026-04-24T23:49:19.578553897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:49:19.581822255Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-24T23:49:22.073341896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:49:22.080030401Z caller=checkpoint.go:378 msg="writing series" size="11 MB" level=debug ts=2026-04-24T23:49:22.08004947Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=5 ts=2026-04-24T23:49:24.773221505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-24T23:49:25.407313513Z caller=checkpoint.go:378 msg="writing series" size="63 kB" level=info ts=2026-04-24T23:49:25.614878256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:49:25.618484992Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:49:27.039684135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:49:27.045174639Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-24T23:49:27.072652071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:49:28.690317091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-24T23:49:28.740443665Z caller=checkpoint.go:378 msg="writing series" size="92 kB" level=info ts=2026-04-24T23:49:31.379837351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:49:31.387809394Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-24T23:49:31.868620759Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:49:31.868902726Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:49:32.061740843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:49:32.065515907Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:49:32.072630825Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:49:32.072650407Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:49:32.072659061Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:49:32.072682729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:49:32.073762293Z caller=checkpoint.go:378 msg="writing series" size="50 kB" level=debug ts=2026-04-24T23:49:35.406923156Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB" level=warn ts=2026-04-24T23:49:37.073318639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:49:37.174594848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:49:37.177757848Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-24T23:49:38.739798957Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB" ts=2026-04-24T23:49:42.031795279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:56938" level=warn ts=2026-04-24T23:49:42.072622598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:49:42.073726836Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=debug ts=2026-04-24T23:49:45.4073184Z caller=checkpoint.go:378 msg="writing series" size="122 kB" level=warn ts=2026-04-24T23:49:47.073458584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:49:48.414677789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:49:48.420275821Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-24T23:49:48.740282525Z caller=checkpoint.go:378 msg="writing series" size="218 kB" level=info ts=2026-04-24T23:49:51.606438275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:49:51.609768924Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-24T23:49:52.072939053Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:49:52.073001421Z caller=checkpoint.go:378 msg="writing series" size="220 kB" ts=2026-04-24T23:49:53.380924707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:40118" level=debug ts=2026-04-24T23:49:54.527398628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-24T23:49:54.774591145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-24T23:49:55.406666945Z caller=checkpoint.go:378 msg="writing series" size="93 kB" level=info ts=2026-04-24T23:49:56.159904346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:49:56.164905708Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-24T23:49:57.072978747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:49:57.427437109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:49:57.430714872Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:49:57.902365963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:49:57.905729369Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-24T23:49:58.706967033Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-24T23:49:58.740367312Z caller=checkpoint.go:378 msg="writing series" size="540 kB" level=debug ts=2026-04-24T23:49:58.740388671Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=11 level=info ts=2026-04-24T23:50:00.665866382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:50:00.669447666Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:50:02.072727174Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-24T23:50:02.0727463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:02.072762203Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:50:02.072773222Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:50:02.07404627Z caller=checkpoint.go:378 msg="writing series" size="529 kB" ts=2026-04-24T23:50:02.190930688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:57570" level=info ts=2026-04-24T23:50:02.316002833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:50:02.320449932Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:50:05.113357406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:50:05.119968805Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-24T23:50:05.407095283Z caller=checkpoint.go:378 msg="writing series" size="415 kB" level=info ts=2026-04-24T23:50:05.903790744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:50:05.907174334Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:50:06.015370123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:50:06.022022888Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:50:06.074373264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:50:06.077467775Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-24T23:50:07.073000618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:50:07.50171455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:50:07.504981135Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-24T23:50:08.740146896Z caller=checkpoint.go:378 msg="writing series" size="158 kB" level=debug ts=2026-04-24T23:50:08.7401722Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3 level=info ts=2026-04-24T23:50:10.795542411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:50:10.802453397Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" ts=2026-04-24T23:50:12.032762041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:44976" level=warn ts=2026-04-24T23:50:12.072722243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:12.073787215Z caller=checkpoint.go:378 msg="writing series" size="41 kB" level=info ts=2026-04-24T23:50:15.204376908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:50:15.20942544Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=debug ts=2026-04-24T23:50:15.407058408Z caller=checkpoint.go:378 msg="writing series" size="106 kB" level=info ts=2026-04-24T23:50:16.538195035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:50:16.541824083Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-24T23:50:17.072917973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:18.740180677Z caller=checkpoint.go:378 msg="writing series" size="129 kB" level=warn ts=2026-04-24T23:50:22.072859057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:22.072972551Z caller=checkpoint.go:378 msg="writing series" size="221 kB" level=info ts=2026-04-24T23:50:23.298210745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:50:23.305047457Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:50:23.305068374Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:23.305099287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:50:23.311819558Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" ts=2026-04-24T23:50:23.383703087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:51256" ts=2026-04-24T23:50:24.776616918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-24T23:50:25.407178083Z caller=checkpoint.go:378 msg="writing series" size="125 kB" level=info ts=2026-04-24T23:50:26.713349661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:50:26.718490849Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-24T23:50:27.072729827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:28.740318062Z caller=checkpoint.go:378 msg="writing series" size="53 kB" level=debug ts=2026-04-24T23:50:30.009703979Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:50:31.868677685Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:50:31.869413429Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=warn ts=2026-04-24T23:50:32.07339747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:50:32.073405556Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:50:32.073428903Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:50:32.073452478Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:50:32.073490893Z caller=checkpoint.go:378 msg="writing series" size="259 kB" level=info ts=2026-04-24T23:50:32.671092429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:50:32.678613791Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-24T23:50:35.407009679Z caller=checkpoint.go:378 msg="writing series" size="19 kB" level=warn ts=2026-04-24T23:50:37.073272552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:38.742431013Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB" level=debug ts=2026-04-24T23:50:38.742454676Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.4 MB" series=9 level=info ts=2026-04-24T23:50:38.873887105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:50:38.880689817Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-24T23:50:42.072436147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:42.075681165Z caller=checkpoint.go:378 msg="writing series" size="3.2 MB" level=debug ts=2026-04-24T23:50:42.075700522Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 MB" series=1 level=info ts=2026-04-24T23:50:43.809708127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:50:43.813251094Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:50:43.813269605Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:43.813290929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:50:43.816276621Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:50:44.277962717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:50:44.281741027Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:50:44.281760601Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:44.281782081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:50:44.284832265Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:50:44.701232339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:50:44.705177477Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:50:44.705196836Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:44.705219641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:50:44.708663708Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-24T23:50:45.407017339Z caller=checkpoint.go:378 msg="writing series" size="909 kB" level=info ts=2026-04-24T23:50:46.392547664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:50:46.396031969Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:50:46.396052768Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:46.396075863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:50:46.398946659Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-24T23:50:47.07281911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:48.739766477Z caller=checkpoint.go:378 msg="writing series" size="23 kB" level=info ts=2026-04-24T23:50:51.475984135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:50:51.482319029Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:50:51.482352226Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:51.482379492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:50:51.489197263Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:50:51.576448043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:50:51.581789976Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:50:51.581815186Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:51.782770479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:50:51.787541983Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-24T23:50:52.073061708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:52.07307939Z caller=checkpoint.go:378 msg="writing series" size="939 B" level=info ts=2026-04-24T23:50:54.28291935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:50:54.28619224Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=error ts=2026-04-24T23:50:54.286212342Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:54.286234632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:50:54.288949685Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:50:54.748187081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:50:54.751843295Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:50:54.751862353Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:54.751882705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:50:54.755060663Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" ts=2026-04-24T23:50:54.778469657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-24T23:50:55.406226736Z caller=checkpoint.go:378 msg="writing series" size="817 B" level=info ts=2026-04-24T23:50:56.656241127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:50:56.65947901Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-24T23:50:57.073073651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:50:58.383835049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-04-24T23:50:58.385366051Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-04-24T23:50:58.389417654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:50:58.397617929Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:50:58.397645875Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:58.397677531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:50:58.40058464Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=debug ts=2026-04-24T23:50:58.739589383Z caller=checkpoint.go:378 msg="writing series" size="174 kB" level=debug ts=2026-04-24T23:50:58.739612569Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5 level=info ts=2026-04-24T23:50:58.832815615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:50:58.836052095Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:50:59.903829055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:50:59.910798044Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-24T23:50:59.938968834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:50:59.942176073Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:50:59.942192451Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:50:59.942212557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:50:59.945065226Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:51:00.67927727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:51:00.682594998Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:51:01.846785473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:51:01.850640515Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:51:01.981113479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:51:01.985489168Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=error ts=2026-04-24T23:51:01.985508302Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:51:01.985528765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:51:01.988281337Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=warn ts=2026-04-24T23:51:02.072718527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:51:02.072724326Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:51:02.072745426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:51:02.072753928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:51:02.0741386Z caller=checkpoint.go:378 msg="writing series" size="806 kB" level=info ts=2026-04-24T23:51:03.585021973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:51:03.593020767Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:51:03.593043122Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:51:03.593067976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:51:03.601307001Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:51:05.256777499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:51:05.260747815Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:51:05.26076781Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:51:05.260789448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:51:05.264080775Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-24T23:51:05.406776854Z caller=checkpoint.go:378 msg="writing series" size="696 kB" level=debug ts=2026-04-24T23:51:05.406798946Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=2 level=info ts=2026-04-24T23:51:05.598077908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:51:05.601892488Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-24T23:51:05.794601723Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:51:05.968275643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:51:05.976430579Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:51:05.976451602Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:51:05.976477959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:51:05.983854966Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:51:07.039923681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:51:07.044933645Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-24T23:51:07.072430427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:51:08.740436963Z caller=checkpoint.go:378 msg="writing series" size="300 kB" level=info ts=2026-04-24T23:51:09.26939611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:51:09.275772756Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:51:09.275792694Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:51:09.275813828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:51:09.280932759Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-24T23:51:10.765569092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:51:10.769217975Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:51:11.037990259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:51:11.045531999Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-24T23:51:12.073195885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:51:12.07338483Z caller=checkpoint.go:378 msg="writing series" size="411 kB" level=info ts=2026-04-24T23:51:13.649738763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:51:13.653557103Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-24T23:51:15.406613062Z caller=checkpoint.go:378 msg="writing series" size="528 B" level=info ts=2026-04-24T23:51:15.848808399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:51:15.852257398Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=error ts=2026-04-24T23:51:15.852273946Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:51:15.852292737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:51:15.855113414Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:51:16.819930283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:51:16.82328524Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-24T23:51:17.073319597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:51:17.689805537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:51:17.693442017Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:51:18.248594194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:51:18.251915974Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=info ts=2026-04-24T23:51:18.661912568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:51:18.665974336Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:51:18.665997188Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:51:18.666018659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:51:18.668927133Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-24T23:51:18.740103376Z caller=checkpoint.go:378 msg="writing series" size="130 kB" level=info ts=2026-04-24T23:51:19.017012618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:51:19.025116309Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-24T23:51:22.073169334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:51:22.073384364Z caller=checkpoint.go:378 msg="writing series" size="410 kB" level=debug ts=2026-04-24T23:51:22.073399834Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5 level=info ts=2026-04-24T23:51:23.030442797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:51:23.038233229Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" ts=2026-04-24T23:51:23.389779259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:43276" level=info ts=2026-04-24T23:51:24.606687633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:51:24.612186559Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" ts=2026-04-24T23:51:24.780823434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=debug ts=2026-04-24T23:51:25.406380945Z caller=checkpoint.go:378 msg="writing series" size="36 kB" level=warn ts=2026-04-24T23:51:27.073127286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:51:28.73975628Z caller=checkpoint.go:378 msg="writing series" size="71 kB" level=info ts=2026-04-24T23:51:30.364305431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:51:30.368192042Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:51:31.723020263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:51:31.729530768Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:51:31.868771822Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:51:31.86918108Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:51:32.07318921Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-24T23:51:32.073202032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:51:32.073219083Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:51:32.073227785Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:51:32.073248203Z caller=checkpoint.go:378 msg="writing series" size="130 kB" level=info ts=2026-04-24T23:51:33.209117411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:51:33.212865616Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-24T23:51:33.816465177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:51:33.822369337Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:51:34.963845971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:51:34.967219946Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:51:35.012267994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:51:35.01535347Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-24T23:51:35.406683317Z caller=checkpoint.go:378 msg="writing series" size="808 kB" level=debug ts=2026-04-24T23:51:35.878067615Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-24T23:51:37.072669941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:51:38.298286738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:51:38.301742884Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:51:38.612424949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:51:38.615700136Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-24T23:51:38.740172856Z caller=checkpoint.go:378 msg="writing series" size="98 kB" level=debug ts=2026-04-24T23:51:38.740196349Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5 level=info ts=2026-04-24T23:51:40.121437939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:51:40.124885478Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:51:40.168091075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:51:40.174542493Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:51:40.218622244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:51:40.222218688Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-24T23:51:42.07345013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:51:42.122869854Z caller=checkpoint.go:378 msg="writing series" size="48 MB" level=debug ts=2026-04-24T23:51:42.122895564Z caller=checkpoint.go:390 msg="flushing series" totalSize="48 MB" series=1 level=info ts=2026-04-24T23:51:42.40921108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:51:42.413746634Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:51:45.082487464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:51:45.087011209Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:51:45.339431825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:51:45.342669896Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-24T23:51:45.407118119Z caller=checkpoint.go:378 msg="writing series" size="676 kB" level=info ts=2026-04-24T23:51:45.668863612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:51:45.675482816Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-24T23:51:47.072683018Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:51:48.740385284Z caller=checkpoint.go:378 msg="writing series" size="468 kB" level=debug ts=2026-04-24T23:51:48.740410894Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=warn ts=2026-04-24T23:51:52.073067054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:51:52.073075799Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB" level=info ts=2026-04-24T23:51:53.576107759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:51:53.579829256Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" ts=2026-04-24T23:51:54.783862583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-24T23:51:54.917881315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:51:54.92325902Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-24T23:51:55.406860739Z caller=checkpoint.go:378 msg="writing series" size="419 kB" level=debug ts=2026-04-24T23:51:56.323359247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-24T23:51:56.949248148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:51:56.957615174Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-24T23:51:57.073206122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:51:58.097370178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:51:58.100937658Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-24T23:51:58.740511142Z caller=checkpoint.go:378 msg="writing series" size="38 kB" level=info ts=2026-04-24T23:52:02.072747719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:52:02.072776387Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:52:02.072784725Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:52:02.072775532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:52:02.073891835Z caller=checkpoint.go:378 msg="writing series" size="6.6 kB" level=debug ts=2026-04-24T23:52:02.285898569Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-24T23:52:05.407110081Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=info ts=2026-04-24T23:52:06.102271428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:52:06.107695172Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-24T23:52:07.073077207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:52:08.740540896Z caller=checkpoint.go:378 msg="writing series" size="271 kB" level=warn ts=2026-04-24T23:52:12.072438667Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:52:12.073570708Z caller=checkpoint.go:378 msg="writing series" size="122 kB" level=debug ts=2026-04-24T23:52:15.406984192Z caller=checkpoint.go:378 msg="writing series" size="61 kB" level=warn ts=2026-04-24T23:52:17.072460461Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:52:18.740302303Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=warn ts=2026-04-24T23:52:22.072795736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:52:22.073947274Z caller=checkpoint.go:378 msg="writing series" size="188 kB" level=debug ts=2026-04-24T23:52:22.073968742Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10 level=info ts=2026-04-24T23:52:22.304663719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:52:22.309966872Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:52:23.250983555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:52:23.254785494Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:52:24.19751146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:52:24.200698208Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:52:24.39689915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:52:24.405121317Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" ts=2026-04-24T23:52:24.787233775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=debug ts=2026-04-24T23:52:25.40666716Z caller=checkpoint.go:378 msg="writing series" size="349 kB" level=info ts=2026-04-24T23:52:26.3581772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:52:26.361873436Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-24T23:52:27.073306392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:52:28.740391167Z caller=checkpoint.go:378 msg="writing series" size="420 kB" level=info ts=2026-04-24T23:52:29.690976526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:52:29.694457447Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=debug ts=2026-04-24T23:52:30.041842806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:52:30.526264296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:52:30.531444652Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=debug ts=2026-04-24T23:52:30.707530758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-24T23:52:31.868669002Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:52:31.869377635Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=warn ts=2026-04-24T23:52:32.073284695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:52:32.073312086Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:52:32.073583798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:52:32.073599272Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:52:32.076231528Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB" level=debug ts=2026-04-24T23:52:32.076249286Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=3 level=info ts=2026-04-24T23:52:32.520244948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:52:32.52724775Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:52:32.946776571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:52:32.950027844Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-24T23:52:35.40702606Z caller=checkpoint.go:378 msg="writing series" size="816 kB" level=info ts=2026-04-24T23:52:36.181914012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:52:36.185566889Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-24T23:52:37.073123087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:52:38.740469017Z caller=checkpoint.go:378 msg="writing series" size="60 kB" level=info ts=2026-04-24T23:52:39.718889275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:52:39.722191276Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" ts=2026-04-24T23:52:42.04658791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42222" level=debug ts=2026-04-24T23:52:42.073400986Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=warn ts=2026-04-24T23:52:42.073401822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:52:43.272068211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:52:43.277771272Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:52:43.972767885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:52:43.979898691Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-24T23:52:44.142387247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:52:44.147789326Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-24T23:52:44.507118457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:52:44.514674735Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:52:45.013085773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:52:45.016791675Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=info ts=2026-04-24T23:52:45.038800575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:52:45.041803069Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-24T23:52:45.406703654Z caller=checkpoint.go:378 msg="writing series" size="14 kB" level=warn ts=2026-04-24T23:52:47.072952674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:52:47.443608522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:52:47.44698967Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-24T23:52:48.740396394Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB" level=warn ts=2026-04-24T23:52:52.072881975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:52:52.072940483Z caller=checkpoint.go:378 msg="writing series" size="17 kB" ts=2026-04-24T23:52:54.790163908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-24T23:52:55.407060632Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB" level=warn ts=2026-04-24T23:52:57.07298569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:52:58.740948338Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB" level=debug ts=2026-04-24T23:52:58.740971869Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=8 level=info ts=2026-04-24T23:53:02.073111893Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:53:02.073145039Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:53:02.073161055Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:53:02.073212746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:53:02.081257917Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014 level=info ts=2026-04-24T23:53:02.08206712Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009238606s level=debug ts=2026-04-24T23:53:04.22731736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-24T23:53:07.07341723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:53:10.275946875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:53:10.282315321Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:53:10.722435781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:53:10.727414066Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-24T23:53:12.073006127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:53:13.593873791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:53:13.601789054Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:53:15.611201982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:53:15.615038432Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:53:15.666220094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:53:15.669214387Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:53:16.116438336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:53:16.119682462Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:53:16.388444797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:53:16.391858996Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:53:16.399907343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:53:16.402856772Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-24T23:53:17.073335314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:53:20.31974842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:53:20.326386862Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-24T23:53:22.07332003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:53:23.324427255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:53:23.328154848Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" ts=2026-04-24T23:53:23.40276848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41876" ts=2026-04-24T23:53:24.793042883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=info ts=2026-04-24T23:53:24.857376063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:53:24.86402807Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-24T23:53:27.073488989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:53:27.240746316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:53:27.24745211Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-24T23:53:28.535615165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:53:28.539043777Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-24T23:53:29.444093651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-04-24T23:53:29.91467879Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:53:31.09590911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:53:31.099859781Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:53:31.868622569Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:53:31.869667312Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:53:32.073257313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=info ts=2026-04-24T23:53:32.073268164Z caller=checkpoint.go:612 msg="starting checkpoint" level=debug ts=2026-04-24T23:53:32.073284292Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:53:32.073293819Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:53:32.073299672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:53:32.073410789Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015 level=debug ts=2026-04-24T23:53:32.074244715Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB" level=debug ts=2026-04-24T23:53:32.074254968Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=1 ts=2026-04-24T23:53:32.206253859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:47178" level=info ts=2026-04-24T23:53:34.028493997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:53:34.032223635Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=debug ts=2026-04-24T23:53:35.40786782Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB" level=warn ts=2026-04-24T23:53:37.073260847Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:53:38.740775612Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB" level=info ts=2026-04-24T23:53:39.117899372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:53:39.121717269Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:53:41.493744142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:53:41.499862961Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-24T23:53:42.072792677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:53:42.073789043Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=info ts=2026-04-24T23:53:43.591383417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:53:43.594886887Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-24T23:53:45.407364483Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=warn ts=2026-04-24T23:53:47.073498282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:53:48.741311378Z caller=checkpoint.go:378 msg="writing series" size="175 kB" level=warn ts=2026-04-24T23:53:52.072881065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:53:52.073967766Z caller=checkpoint.go:378 msg="writing series" size="14 kB" ts=2026-04-24T23:53:54.795701413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-24T23:53:55.407399558Z caller=checkpoint.go:378 msg="writing series" size="25 kB" level=debug ts=2026-04-24T23:53:55.657158904Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:53:56.2852964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:53:56.292806487Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-24T23:53:57.07335933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:53:58.740784957Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB" level=info ts=2026-04-24T23:54:02.072733654Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-24T23:54:02.072754828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:02.072769458Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:54:02.072780017Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:54:02.073897946Z caller=checkpoint.go:378 msg="writing series" size="92 kB" level=info ts=2026-04-24T23:54:03.809090956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:54:03.81269563Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:54:04.511756495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:54:04.515098246Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-24T23:54:05.408433014Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB" level=debug ts=2026-04-24T23:54:05.408463251Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=10 level=info ts=2026-04-24T23:54:05.949372486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:54:05.956275695Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:54:06.983508839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:54:06.986587847Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=warn ts=2026-04-24T23:54:07.073121521Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:07.654478209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-04-24T23:54:08.740580714Z caller=checkpoint.go:378 msg="writing series" size="171 kB" level=info ts=2026-04-24T23:54:10.083648695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:54:10.087061602Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:54:10.597358423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:54:10.60073598Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:54:11.374064017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:54:11.381615833Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:54:11.981281032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:54:11.990587093Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-24T23:54:12.073012932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:12.074176365Z caller=checkpoint.go:378 msg="writing series" size="97 kB" level=info ts=2026-04-24T23:54:13.858663375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:54:13.861984406Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:54:14.750259491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:54:14.753792048Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-24T23:54:15.407177415Z caller=checkpoint.go:378 msg="writing series" size="42 kB" level=warn ts=2026-04-24T23:54:17.073340821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:18.740668727Z caller=checkpoint.go:378 msg="writing series" size="389 kB" level=info ts=2026-04-24T23:54:19.168074457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:54:19.171765062Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:54:19.386757496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:54:19.393591001Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-24T23:54:20.464348122Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-24T23:54:22.073068811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:22.083462882Z caller=checkpoint.go:378 msg="writing series" size="11 MB" level=debug ts=2026-04-24T23:54:22.083486896Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=5 ts=2026-04-24T23:54:24.798426825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-24T23:54:25.407132938Z caller=checkpoint.go:378 msg="writing series" size="63 kB" level=info ts=2026-04-24T23:54:25.448448728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:54:25.454044595Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-24T23:54:26.41287945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:54:26.416378906Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=warn ts=2026-04-24T23:54:27.07278846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:54:28.143300107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:54:28.147152652Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-24T23:54:28.741267002Z caller=checkpoint.go:378 msg="writing series" size="92 kB" level=info ts=2026-04-24T23:54:31.868730683Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:54:31.86873642Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=warn ts=2026-04-24T23:54:32.072865228Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:54:32.072865318Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:54:32.07289544Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:54:32.072904925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:54:32.073966798Z caller=checkpoint.go:378 msg="writing series" size="50 kB" level=info ts=2026-04-24T23:54:34.115283188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:54:34.118890536Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-24T23:54:34.152008032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:54:34.155026258Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-24T23:54:35.407485527Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB" level=warn ts=2026-04-24T23:54:37.072694937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:38.741071683Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB" ts=2026-04-24T23:54:42.057355514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36496" level=warn ts=2026-04-24T23:54:42.073175737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:42.074280209Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=debug ts=2026-04-24T23:54:45.407902156Z caller=checkpoint.go:378 msg="writing series" size="122 kB" level=info ts=2026-04-24T23:54:45.871173825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:54:45.880282742Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-24T23:54:47.073119775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:48.741297907Z caller=checkpoint.go:378 msg="writing series" size="232 kB" level=info ts=2026-04-24T23:54:49.828859595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:54:49.835931416Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:54:51.856365664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:54:51.859814571Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-24T23:54:51.887593904Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-24T23:54:52.073297721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:52.074471832Z caller=checkpoint.go:378 msg="writing series" size="220 kB" level=debug ts=2026-04-24T23:54:54.106974019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-24T23:54:54.108517958Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-24T23:54:54.801006834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=info ts=2026-04-24T23:54:54.866511074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:54:54.869837671Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-24T23:54:55.407179145Z caller=checkpoint.go:378 msg="writing series" size="97 kB" level=info ts=2026-04-24T23:54:56.413276339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:54:56.416905937Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-24T23:54:57.072908549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:54:58.741133595Z caller=checkpoint.go:378 msg="writing series" size="401 kB" level=debug ts=2026-04-24T23:54:58.74115851Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=11 level=info ts=2026-04-24T23:55:01.383857607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:55:01.387399914Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:55:02.073159649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-24T23:55:02.073175281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:55:02.073191749Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:55:02.073200316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:55:02.074576002Z caller=checkpoint.go:378 msg="writing series" size="529 kB" ts=2026-04-24T23:55:02.214048114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:52798" level=info ts=2026-04-24T23:55:03.801568441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:55:03.804766813Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:55:04.433983618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:55:04.437353057Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-24T23:55:05.407353177Z caller=checkpoint.go:378 msg="writing series" size="415 kB" level=info ts=2026-04-24T23:55:05.951763155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:55:05.95882029Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-24T23:55:07.073153125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:55:07.593008247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:55:07.600663168Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:55:08.026882297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:55:08.030135653Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-24T23:55:08.741350669Z caller=checkpoint.go:378 msg="writing series" size="204 kB" level=debug ts=2026-04-24T23:55:08.741375938Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3 level=info ts=2026-04-24T23:55:09.3414326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:55:09.345665027Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:55:11.150553609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:55:11.155585574Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-24T23:55:12.072440952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:55:12.074607355Z caller=checkpoint.go:378 msg="writing series" size="41 kB" level=info ts=2026-04-24T23:55:13.37207681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="377 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:55:13.379600354Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-24T23:55:15.40700975Z caller=checkpoint.go:378 msg="writing series" size="106 kB" level=info ts=2026-04-24T23:55:16.876428473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:55:16.882350547Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-24T23:55:17.072918735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:55:17.150141176Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-24T23:55:18.740840748Z caller=checkpoint.go:378 msg="writing series" size="137 kB" level=info ts=2026-04-24T23:55:19.37045674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:55:19.373717158Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=warn ts=2026-04-24T23:55:22.073066208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:55:22.074278099Z caller=checkpoint.go:378 msg="writing series" size="221 kB" level=info ts=2026-04-24T23:55:24.146000934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:55:24.150698517Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" ts=2026-04-24T23:55:24.803463598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-24T23:55:25.407593867Z caller=checkpoint.go:378 msg="writing series" size="125 kB" level=warn ts=2026-04-24T23:55:27.072780928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:55:28.74113431Z caller=checkpoint.go:378 msg="writing series" size="79 kB" level=info ts=2026-04-24T23:55:30.270171701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:55:30.274138833Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:55:31.828862447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:55:31.834444297Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:55:31.868720992Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:55:31.869093676Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=warn ts=2026-04-24T23:55:32.07253018Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:55:32.072537932Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:55:32.07256068Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:55:32.072569285Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:55:32.073776022Z caller=checkpoint.go:378 msg="writing series" size="261 kB" ts=2026-04-24T23:55:32.217699824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41854" level=debug ts=2026-04-24T23:55:35.406979968Z caller=checkpoint.go:378 msg="writing series" size="19 kB" level=warn ts=2026-04-24T23:55:37.072901273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:55:37.749203293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:55:37.756938306Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-24T23:55:38.743612877Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB" level=debug ts=2026-04-24T23:55:38.743635708Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.5 MB" series=9 level=warn ts=2026-04-24T23:55:42.07308685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:55:42.07596918Z caller=checkpoint.go:378 msg="writing series" size="3.2 MB" level=debug ts=2026-04-24T23:55:42.07598461Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 MB" series=1 level=info ts=2026-04-24T23:55:43.50036869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:55:43.50417548Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:55:45.035138387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:55:45.038770301Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-24T23:55:45.409189018Z caller=checkpoint.go:378 msg="writing series" size="909 kB" level=debug ts=2026-04-24T23:55:46.713500653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=warn ts=2026-04-24T23:55:47.072929315Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:55:47.541138376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:55:47.551447593Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:55:48.079163884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:55:48.082607794Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-24T23:55:48.741043443Z caller=checkpoint.go:378 msg="writing series" size="23 kB" level=debug ts=2026-04-24T23:55:50.735815107Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:55:51.106972231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:55:51.11080212Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-24T23:55:52.073047388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:55:52.07415505Z caller=checkpoint.go:378 msg="writing series" size="939 B" ts=2026-04-24T23:55:53.413552064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:34568" level=info ts=2026-04-24T23:55:53.550174203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:55:53.553608946Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" ts=2026-04-24T23:55:54.806096226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-24T23:55:55.142661144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:55:55.146028045Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-24T23:55:55.407436895Z caller=checkpoint.go:378 msg="writing series" size="817 B" level=info ts=2026-04-24T23:55:56.434350567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:55:56.43805162Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-24T23:55:57.073112701Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:55:58.740757189Z caller=checkpoint.go:378 msg="writing series" size="198 kB" level=debug ts=2026-04-24T23:55:58.740781204Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5 level=info ts=2026-04-24T23:55:59.087726639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:55:59.091951857Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:55:59.126011128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:55:59.134017212Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:56:01.4218885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:56:01.427628462Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-24T23:56:02.073176945Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:56:02.073173792Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:56:02.073204012Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:56:02.073214403Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:56:02.075073781Z caller=checkpoint.go:378 msg="writing series" size="604 kB" ts=2026-04-24T23:56:02.22134578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:56434" level=info ts=2026-04-24T23:56:03.984513418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:56:03.988294387Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:56:05.118442124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:56:05.122123764Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-24T23:56:05.407440248Z caller=checkpoint.go:378 msg="writing series" size="696 kB" level=debug ts=2026-04-24T23:56:05.407464979Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2 level=warn ts=2026-04-24T23:56:07.073424586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:56:08.210064774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:56:08.21406136Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=debug ts=2026-04-24T23:56:08.740674937Z caller=checkpoint.go:378 msg="writing series" size="107 kB" level=info ts=2026-04-24T23:56:10.038398737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:56:10.046607427Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-24T23:56:12.073222985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:56:12.074402976Z caller=checkpoint.go:378 msg="writing series" size="411 kB" level=debug ts=2026-04-24T23:56:15.407398364Z caller=checkpoint.go:378 msg="writing series" size="528 B" level=warn ts=2026-04-24T23:56:17.073443125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:56:18.741172491Z caller=checkpoint.go:378 msg="writing series" size="130 kB" level=info ts=2026-04-24T23:56:19.5818446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:56:19.589762447Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:56:19.801526023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:56:19.805378115Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:56:19.919838805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:56:19.927954044Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-24T23:56:22.073222276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:56:22.07445419Z caller=checkpoint.go:378 msg="writing series" size="410 kB" level=debug ts=2026-04-24T23:56:22.07447413Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5 ts=2026-04-24T23:56:24.808923237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-24T23:56:25.39221964Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:56:25.392283769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="36 kB" avg_uncomp="36 kB" idle=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-24T23:56:25.397291367Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-24T23:56:25.407468287Z caller=checkpoint.go:378 msg="writing series" size="36 kB" level=warn ts=2026-04-24T23:56:27.073500901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:56:28.740525519Z caller=checkpoint.go:378 msg="writing series" size="76 kB" level=info ts=2026-04-24T23:56:31.868778158Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:56:31.869179444Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:56:32.073216132Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:56:32.073255661Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:56:32.073266604Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:56:32.073278563Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:56:32.074451472Z caller=checkpoint.go:378 msg="writing series" size="136 kB" level=debug ts=2026-04-24T23:56:34.191651885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-24T23:56:34.499900027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:56:34.503125651Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:56:34.981101522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:56:34.984422297Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-24T23:56:35.408307411Z caller=checkpoint.go:378 msg="writing series" size="952 kB" level=debug ts=2026-04-24T23:56:35.408352143Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4 level=warn ts=2026-04-24T23:56:37.072606415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:56:38.741136135Z caller=checkpoint.go:378 msg="writing series" size="98 kB" level=warn ts=2026-04-24T23:56:42.073368069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:56:42.107592022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:56:42.11144861Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-24T23:56:42.13430295Z caller=checkpoint.go:378 msg="writing series" size="49 MB" level=debug ts=2026-04-24T23:56:42.13433634Z caller=checkpoint.go:390 msg="flushing series" totalSize="49 MB" series=2 level=info ts=2026-04-24T23:56:43.476724803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-24T23:56:43.480222425Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-24T23:56:43.820047949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:56:43.82764272Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:56:44.189720354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:56:44.19352267Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:56:44.243577975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:56:44.246949618Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:56:45.072161134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:56:45.076260861Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-24T23:56:45.407703259Z caller=checkpoint.go:378 msg="writing series" size="676 kB" level=info ts=2026-04-24T23:56:46.639739843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:56:46.643216368Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-24T23:56:47.072883977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:56:48.741008122Z caller=checkpoint.go:378 msg="writing series" size="468 kB" level=debug ts=2026-04-24T23:56:48.741035242Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=info ts=2026-04-24T23:56:50.790347537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:56:50.798831042Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-24T23:56:52.073288118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:56:52.07437965Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB" level=info ts=2026-04-24T23:56:52.084494214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:56:52.087972525Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-24T23:56:53.378965833Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 ts=2026-04-24T23:56:54.811799918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-24T23:56:55.407713245Z caller=checkpoint.go:378 msg="writing series" size="525 kB" level=info ts=2026-04-24T23:56:55.875177038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:56:55.883636739Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:56:55.88365641Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:56:55.883688856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:56:55.890100218Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-24T23:56:57.073489398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:56:58.117430702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:56:58.124367196Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-24T23:56:58.740982854Z caller=checkpoint.go:378 msg="writing series" size="13 kB" level=info ts=2026-04-24T23:56:58.945738634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:56:58.949285583Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-24T23:56:59.156883592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:56:59.161912615Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-24T23:57:00.970596191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:57:00.977431989Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:57:00.97745348Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:00.977479091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:57:00.984685034Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-24T23:57:02.073180083Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-24T23:57:02.073188986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:57:02.073215556Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:57:02.073226915Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:57:02.074317735Z caller=checkpoint.go:378 msg="writing series" size="6.6 kB" level=info ts=2026-04-24T23:57:02.158451848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:57:02.161765098Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" ts=2026-04-24T23:57:02.226283914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:47866" level=info ts=2026-04-24T23:57:02.411589909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:57:02.41486206Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=info ts=2026-04-24T23:57:05.065508455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:57:05.07022961Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:57:05.070248822Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:05.070273671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:57:05.074558299Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=debug ts=2026-04-24T23:57:05.40762425Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=warn ts=2026-04-24T23:57:07.073191931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:57:08.741092752Z caller=checkpoint.go:378 msg="writing series" size="76 kB" level=info ts=2026-04-24T23:57:09.333666494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:57:09.341227003Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-24T23:57:12.073500091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:57:12.074622143Z caller=checkpoint.go:378 msg="writing series" size="145 kB" level=debug ts=2026-04-24T23:57:15.407587736Z caller=checkpoint.go:378 msg="writing series" size="63 kB" level=info ts=2026-04-24T23:57:16.107634618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:57:16.115351938Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-24T23:57:17.072750502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:57:17.965817227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:57:17.969640676Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:57:17.96966371Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:17.969686169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:57:17.972645289Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-24T23:57:18.740708467Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=debug ts=2026-04-24T23:57:19.086036674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:57:21.038779022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-24T23:57:21.042248462Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-24T23:57:22.073355408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:57:22.07448716Z caller=checkpoint.go:378 msg="writing series" size="188 kB" level=debug ts=2026-04-24T23:57:23.25776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-24T23:57:23.560602216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:57:23.5639022Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=error ts=2026-04-24T23:57:23.563921824Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:23.563941959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:57:23.566882024Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:57:24.674161214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:57:24.67903273Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" ts=2026-04-24T23:57:24.813494152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-24T23:57:25.407483032Z caller=checkpoint.go:378 msg="writing series" size="438 kB" level=debug ts=2026-04-24T23:57:25.407508452Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=11 level=warn ts=2026-04-24T23:57:27.072442806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:57:28.740796992Z caller=checkpoint.go:378 msg="writing series" size="573 kB" level=info ts=2026-04-24T23:57:29.366782961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:57:29.37070581Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:57:29.370723391Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:29.370745545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:57:29.373578465Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:57:30.039781653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:57:30.04353972Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:57:30.043570544Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:30.043594335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:57:30.046458766Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:57:31.577296237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:57:31.580842652Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-24T23:57:31.868609636Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:57:31.868614721Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-24T23:57:32.072719067Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:57:32.072748397Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:57:32.072759896Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:57:32.07275998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:57:32.074772762Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB" level=debug ts=2026-04-24T23:57:32.074785563Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=2 level=info ts=2026-04-24T23:57:32.183112534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:57:32.186747994Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:57:32.186765924Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:32.186786658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:57:32.189608353Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:57:32.200685507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:57:32.20413084Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:57:32.204145014Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:32.487594918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:57:32.491565191Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:57:34.492354232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:57:34.495737617Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:57:34.49575818Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:34.495779125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:57:34.498508386Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-24T23:57:35.408176814Z caller=checkpoint.go:378 msg="writing series" size="816 kB" level=info ts=2026-04-24T23:57:36.138936621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:57:36.145563564Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-24T23:57:37.073135955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:57:38.741239301Z caller=checkpoint.go:378 msg="writing series" size="61 kB" level=info ts=2026-04-24T23:57:39.865712321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:57:39.869057351Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:57:40.056459579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:57:40.063284567Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-24T23:57:40.176672763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:57:40.180051845Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:57:40.230056171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:57:40.233314817Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:57:40.233346383Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:40.233367619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:57:40.23630809Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-24T23:57:41.220106427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:57:41.22667639Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:57:41.226699957Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:41.2267242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:57:41.232839068Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-24T23:57:41.252941842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:57:41.256033508Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-24T23:57:42.072637773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:57:42.073662463Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=info ts=2026-04-24T23:57:42.461106786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:57:42.464407481Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:57:43.242352623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:57:43.246161286Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-24T23:57:45.40726984Z caller=checkpoint.go:378 msg="writing series" size="15 kB" level=info ts=2026-04-24T23:57:46.468878308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:57:46.472628963Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:57:46.472649048Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:46.472671843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:57:46.475605866Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-24T23:57:47.073170197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:57:47.345834157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:57:47.356605126Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=debug ts=2026-04-24T23:57:48.740622589Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB" level=debug ts=2026-04-24T23:57:49.829246423Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:57:50.410854843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:57:50.414592894Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:57:51.098658788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:57:51.103375875Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:57:51.10339804Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:51.103419563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:57:51.106536071Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-24T23:57:52.072770635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:57:52.07386511Z caller=checkpoint.go:378 msg="writing series" size="17 kB" level=info ts=2026-04-24T23:57:52.854381987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:57:52.861412783Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" ts=2026-04-24T23:57:53.420428923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:54978" level=info ts=2026-04-24T23:57:53.544185703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:57:53.549512913Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:57:53.549533141Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:53.549555748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:57:53.554571059Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-24T23:57:54.286031651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:57:54.289394194Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" ts=2026-04-24T23:57:54.81606646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-24T23:57:55.090466524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-24T23:57:55.094192443Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=error ts=2026-04-24T23:57:55.094210568Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:55.094231504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:57:55.097991371Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=debug ts=2026-04-24T23:57:55.407233268Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB" level=info ts=2026-04-24T23:57:56.020008221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:57:56.023256466Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=error ts=2026-04-24T23:57:56.023275555Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:56.023296845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:57:56.02596072Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-24T23:57:57.07278525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:57:57.658373313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:57:57.665075294Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:57:57.665096346Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-24T23:57:57.665121238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:57:57.671039927Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-24T23:57:58.741620603Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB" level=debug ts=2026-04-24T23:57:58.74164391Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=8 level=info ts=2026-04-24T23:58:02.073344487Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:58:02.073378138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:58:02.073391364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:58:02.073392212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:58:02.081255993Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015 level=info ts=2026-04-24T23:58:02.082304946Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008758522s ts=2026-04-24T23:58:02.232456468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:40410" level=info ts=2026-04-24T23:58:02.44498316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:58:02.449716998Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:58:04.886510147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:58:04.890163248Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:58:06.80315779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:58:06.806683332Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-24T23:58:07.073353963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:58:07.321574579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-24T23:58:07.370099898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:58:07.373735805Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:58:08.505469133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:58:08.510556297Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-24T23:58:12.072950069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:58:12.077768852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:45078" level=info ts=2026-04-24T23:58:12.610977762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:58:12.61444973Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-24T23:58:12.987583199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:58:12.991022039Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-24T23:58:13.616566649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:58:13.620770026Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=debug ts=2026-04-24T23:58:14.133949345Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:58:14.402595279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:58:14.408896171Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-24T23:58:16.186829219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:58:16.190165627Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-24T23:58:17.072944224Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:58:17.361851967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:58:17.365040799Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:58:17.607064278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-24T23:58:17.610394377Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-24T23:58:17.685702655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:58:17.688804166Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-24T23:58:18.563105852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:58:18.567195295Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:58:20.452373903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:58:20.458891243Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-24T23:58:22.072698383Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-24T23:58:24.818764753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=info ts=2026-04-24T23:58:25.890660952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:58:25.895899439Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-24T23:58:27.073385181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:58:31.635945134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:58:31.642842199Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-24T23:58:31.868263163Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-24T23:58:31.869253327Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-24T23:58:32.073229835Z caller=checkpoint.go:612 msg="starting checkpoint" level=info ts=2026-04-24T23:58:32.073248451Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:58:32.073279051Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:58:32.073287204Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:58:32.073303444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:58:32.073411575Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016 level=debug ts=2026-04-24T23:58:32.075275334Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB" level=debug ts=2026-04-24T23:58:32.075287449Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=1 level=info ts=2026-04-24T23:58:34.254138122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:58:34.259676223Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-24T23:58:35.219786663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:58:35.22734577Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-24T23:58:35.407106806Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB" level=warn ts=2026-04-24T23:58:37.073367267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:58:38.740274398Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB" level=info ts=2026-04-24T23:58:39.213609389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:58:39.217445482Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-24T23:58:40.513154269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:58:40.517734152Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-24T23:58:41.477928559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:58:41.483735743Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-24T23:58:42.072432881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:58:42.074579363Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" ts=2026-04-24T23:58:42.079007561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:46774" level=info ts=2026-04-24T23:58:44.50318764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:58:44.507170728Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-24T23:58:45.407075858Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=info ts=2026-04-24T23:58:45.61771197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:58:45.621335004Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-24T23:58:45.772964051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-24T23:58:47.072936854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:58:48.741044965Z caller=checkpoint.go:378 msg="writing series" size="232 kB" level=info ts=2026-04-24T23:58:51.030645559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:58:51.037408442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-24T23:58:52.07317142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:58:52.074273187Z caller=checkpoint.go:378 msg="writing series" size="15 kB" level=debug ts=2026-04-24T23:58:53.056157015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-24T23:58:53.057836805Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-24T23:58:53.426795431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:60000" level=info ts=2026-04-24T23:58:54.822078684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" ts=2026-04-24T23:58:54.822192442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=error ts=2026-04-24T23:58:54.825982719Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-24T23:58:55.407691382Z caller=checkpoint.go:378 msg="writing series" size="25 kB" level=warn ts=2026-04-24T23:58:57.073301207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:58:58.740518155Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB" level=info ts=2026-04-24T23:59:01.975774411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:59:01.979279947Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:59:02.072629472Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:59:02.072664498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:59:02.072673299Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-24T23:59:02.072718492Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:02.07383603Z caller=checkpoint.go:378 msg="writing series" size="202 kB" level=info ts=2026-04-24T23:59:03.677641361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:59:03.681815467Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-24T23:59:04.77415267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:59:04.782960519Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-24T23:59:05.407745092Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB" level=debug ts=2026-04-24T23:59:05.407768514Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.0 MB" series=10 level=info ts=2026-04-24T23:59:05.629628754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:59:05.633290081Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-24T23:59:07.072719536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:08.741189346Z caller=checkpoint.go:378 msg="writing series" size="181 kB" level=info ts=2026-04-24T23:59:09.819750379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-24T23:59:09.823967811Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=warn ts=2026-04-24T23:59:12.07306809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:12.074129217Z caller=checkpoint.go:378 msg="writing series" size="247 kB" ts=2026-04-24T23:59:12.080559613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36248" level=info ts=2026-04-24T23:59:13.083722636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-24T23:59:13.086980243Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-24T23:59:13.415767249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-24T23:59:13.419188184Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-24T23:59:13.985015474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-24T23:59:13.988731836Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-24T23:59:14.609548985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-24T23:59:14.616368198Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-24T23:59:15.406968365Z caller=checkpoint.go:378 msg="writing series" size="43 kB" level=info ts=2026-04-24T23:59:16.126186535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-24T23:59:16.129755285Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-24T23:59:17.07293163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:18.741415556Z caller=checkpoint.go:378 msg="writing series" size="460 kB" level=info ts=2026-04-24T23:59:19.6617249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-24T23:59:19.668467013Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-24T23:59:20.380530898Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-24T23:59:20.71451791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-24T23:59:20.719870358Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-24T23:59:22.073341263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:22.082377035Z caller=checkpoint.go:378 msg="writing series" size="12 MB" level=debug ts=2026-04-24T23:59:22.082397006Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=5 level=info ts=2026-04-24T23:59:22.56423361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-24T23:59:22.568980734Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" ts=2026-04-24T23:59:24.82489029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-24T23:59:25.407411212Z caller=checkpoint.go:378 msg="writing series" size="63 kB" level=info ts=2026-04-24T23:59:25.721371871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-24T23:59:25.726970573Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-24T23:59:27.072966711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:28.740940096Z caller=checkpoint.go:378 msg="writing series" size="97 kB" level=info ts=2026-04-24T23:59:29.992070432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-24T23:59:29.998581459Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-24T23:59:31.868855237Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-24T23:59:31.869038235Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=warn ts=2026-04-24T23:59:32.072900014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:59:32.072906925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-24T23:59:32.072930366Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-24T23:59:32.072938843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-24T23:59:32.074033902Z caller=checkpoint.go:378 msg="writing series" size="51 kB" level=debug ts=2026-04-24T23:59:35.407416362Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB" level=warn ts=2026-04-24T23:59:37.072430635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:38.143261537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-04-24T23:59:38.740888533Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB" level=info ts=2026-04-24T23:59:41.991641204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-24T23:59:41.9963143Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-24T23:59:42.072665857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:42.073739703Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" ts=2026-04-24T23:59:42.081906142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60250" level=info ts=2026-04-24T23:59:43.469703456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-24T23:59:43.47353501Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-24T23:59:45.407316116Z caller=checkpoint.go:378 msg="writing series" size="122 kB" level=info ts=2026-04-24T23:59:46.407165516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-24T23:59:46.411108149Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-24T23:59:46.717547105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-24T23:59:46.72548234Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-24T23:59:47.072845871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-24T23:59:47.566366442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-24T23:59:47.569898948Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-24T23:59:48.741210101Z caller=checkpoint.go:378 msg="writing series" size="232 kB" level=info ts=2026-04-24T23:59:50.870728169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-24T23:59:50.87746888Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-24T23:59:52.073083958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:52.074193525Z caller=checkpoint.go:378 msg="writing series" size="220 kB" level=info ts=2026-04-24T23:59:52.780274253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-24T23:59:52.784145052Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-24T23:59:53.288661793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-24T23:59:53.292082776Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-24T23:59:54.359400602Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 ts=2026-04-24T23:59:54.826886884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-24T23:59:55.408017014Z caller=checkpoint.go:378 msg="writing series" size="103 kB" level=warn ts=2026-04-24T23:59:57.073148029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-24T23:59:58.741049626Z caller=checkpoint.go:378 msg="writing series" size="534 kB" level=debug ts=2026-04-24T23:59:58.741073545Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=11 level=info ts=2026-04-25T00:00:02.073132585Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:00:02.073166061Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:00:02.073175252Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:00:02.073227683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:00:02.074559292Z caller=checkpoint.go:378 msg="writing series" size="529 kB" level=info ts=2026-04-25T00:00:04.938704862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:00:04.947150298Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-25T00:00:05.378263964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:00:05.385989833Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-25T00:00:05.407268143Z caller=checkpoint.go:378 msg="writing series" size="415 kB" level=info ts=2026-04-25T00:00:06.322300916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:00:06.326539217Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-25T00:00:07.072847859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:00:07.497067436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:00:07.501014756Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:00:07.918867714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:00:07.923086642Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:00:08.741443248Z caller=checkpoint.go:378 msg="writing series" size="253 kB" level=debug ts=2026-04-25T00:00:08.741465833Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3 level=info ts=2026-04-25T00:00:09.977594958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:00:09.985431448Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-25T00:00:12.073209869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:00:12.074307979Z caller=checkpoint.go:378 msg="writing series" size="41 kB" level=info ts=2026-04-25T00:00:13.189368365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:00:13.192968294Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-25T00:00:15.40712188Z caller=checkpoint.go:378 msg="writing series" size="109 kB" level=info ts=2026-04-25T00:00:16.954300882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:00:16.959840643Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=warn ts=2026-04-25T00:00:17.073421075Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:00:18.440894929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:00:18.444875462Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=debug ts=2026-04-25T00:00:18.740907731Z caller=checkpoint.go:378 msg="writing series" size="146 kB" level=info ts=2026-04-25T00:00:19.185910625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:00:19.191716705Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:00:22.072894699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:00:22.074090573Z caller=checkpoint.go:378 msg="writing series" size="221 kB" level=info ts=2026-04-25T00:00:22.471310448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:00:22.476519974Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" ts=2026-04-25T00:00:24.829098286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:00:25.407393292Z caller=checkpoint.go:378 msg="writing series" size="125 kB" level=debug ts=2026-04-25T00:00:26.04120782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:00:27.073145339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:00:28.390065192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:00:28.394386026Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-25T00:00:28.741299943Z caller=checkpoint.go:378 msg="writing series" size="103 kB" level=info ts=2026-04-25T00:00:31.869137698Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:00:31.86955401Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=debug ts=2026-04-25T00:00:31.870821115Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads" level=debug ts=2026-04-25T00:00:31.870877964Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974527 n=1 level=debug ts=2026-04-25T00:00:31.870937175Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974527 n=1 level=info ts=2026-04-25T00:00:32.072934055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:00:32.072976619Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:00:32.072988197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:00:32.07298103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:00:32.074102584Z caller=checkpoint.go:378 msg="writing series" size="49 kB" ts=2026-04-25T00:00:32.245669344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39722" level=info ts=2026-04-25T00:00:32.308481223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:00:32.316312964Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:00:32.349374914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:00:32.352838976Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:00:33.767351795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:00:33.771679008Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:00:34.081569977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-04-25T00:00:35.406980557Z caller=checkpoint.go:378 msg="writing series" size="19 kB" level=warn ts=2026-04-25T00:00:37.072652802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:00:38.744522825Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB" level=debug ts=2026-04-25T00:00:38.744544884Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.3 MB" series=9 level=info ts=2026-04-25T00:00:40.407890065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:00:40.411429608Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:00:41.376139742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:00:41.379561742Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:00:42.072723948Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:00:42.075552852Z caller=checkpoint.go:378 msg="writing series" size="3.5 MB" level=debug ts=2026-04-25T00:00:42.075568705Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.5 MB" series=1 ts=2026-04-25T00:00:42.086367709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:40452" level=info ts=2026-04-25T00:00:45.115836663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:00:45.123442121Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:00:45.402145043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=debug ts=2026-04-25T00:00:45.407674534Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB" level=error ts=2026-04-25T00:00:45.409649265Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:00:47.073470253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:00:48.74060491Z caller=checkpoint.go:378 msg="writing series" size="23 kB" level=debug ts=2026-04-25T00:00:48.740638102Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=info ts=2026-04-25T00:00:51.28250574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:00:51.286400174Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:00:52.073496123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:00:52.074530288Z caller=checkpoint.go:378 msg="writing series" size="939 B" level=info ts=2026-04-25T00:00:52.294442323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:00:52.298717671Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" ts=2026-04-25T00:00:53.435679565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:52626" ts=2026-04-25T00:00:54.831573301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:00:55.407085588Z caller=checkpoint.go:378 msg="writing series" size="817 B" level=warn ts=2026-04-25T00:00:57.073248203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:00:58.74126749Z caller=checkpoint.go:378 msg="writing series" size="205 kB" level=info ts=2026-04-25T00:00:59.540722094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:00:59.545024307Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:01:00.020547164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:01:00.029229355Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-25T00:01:00.753355196Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:01:01.080474968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:01:01.084288417Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:01:01.093370903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:01:01.096419023Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:01:02.07285175Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:01:02.072885215Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:01:02.072894336Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:01:02.072890839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:02.074600673Z caller=checkpoint.go:378 msg="writing series" size="667 kB" level=info ts=2026-04-25T00:01:03.708079988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:01:03.711622036Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=debug ts=2026-04-25T00:01:05.407977945Z caller=checkpoint.go:378 msg="writing series" size="609 kB" level=debug ts=2026-04-25T00:01:05.408000905Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5 level=warn ts=2026-04-25T00:01:07.073071674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:08.741007971Z caller=checkpoint.go:378 msg="writing series" size="159 kB" level=info ts=2026-04-25T00:01:11.030995454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:01:11.036087463Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=debug ts=2026-04-25T00:01:11.550792902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=warn ts=2026-04-25T00:01:12.073074387Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:12.074401425Z caller=checkpoint.go:378 msg="writing series" size="411 kB" level=info ts=2026-04-25T00:01:12.92022814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:01:12.92587155Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:01:12.974067254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:01:12.97850809Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-25T00:01:14.655877623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:01:14.660056459Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-25T00:01:15.40777111Z caller=checkpoint.go:378 msg="writing series" size="528 B" level=info ts=2026-04-25T00:01:16.447386372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:01:16.451090596Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:01:16.676089019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:01:16.679758505Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-25T00:01:17.073447265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:18.740684119Z caller=checkpoint.go:378 msg="writing series" size="130 kB" level=warn ts=2026-04-25T00:01:22.072744485Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:22.074018137Z caller=checkpoint.go:378 msg="writing series" size="410 kB" level=debug ts=2026-04-25T00:01:22.074039715Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5 level=info ts=2026-04-25T00:01:22.611268976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:01:22.619635154Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" ts=2026-04-25T00:01:24.833654008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:01:25.406955255Z caller=checkpoint.go:378 msg="writing series" size="36 kB" level=info ts=2026-04-25T00:01:26.441929545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:01:26.449674399Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:01:27.072724864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:01:27.526902036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:01:27.531448915Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:01:27.791952917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:01:27.797064744Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:01:28.741242631Z caller=checkpoint.go:378 msg="writing series" size="79 kB" level=info ts=2026-04-25T00:01:30.158805911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:01:30.162737863Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:01:31.868340099Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:01:31.869347029Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-25T00:01:32.072925662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:01:32.072958567Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:01:32.072967317Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:01:32.072928542Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:32.07424803Z caller=checkpoint.go:378 msg="writing series" size="143 kB" level=debug ts=2026-04-25T00:01:33.821161205Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:01:34.178164721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:01:34.185488141Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-25T00:01:35.407928697Z caller=checkpoint.go:378 msg="writing series" size="866 kB" level=debug ts=2026-04-25T00:01:35.407953818Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4 level=warn ts=2026-04-25T00:01:37.072452092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:38.741159238Z caller=checkpoint.go:378 msg="writing series" size="116 kB" level=info ts=2026-04-25T00:01:41.358093141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:01:41.368740909Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-25T00:01:42.07330373Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:42.104693278Z caller=checkpoint.go:378 msg="writing series" size="49 MB" level=debug ts=2026-04-25T00:01:42.104715549Z caller=checkpoint.go:390 msg="flushing series" totalSize="49 MB" series=2 level=info ts=2026-04-25T00:01:42.609776963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:01:42.61368805Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:01:44.124070732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:01:44.127942805Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-25T00:01:45.408047252Z caller=checkpoint.go:378 msg="writing series" size="676 kB" level=info ts=2026-04-25T00:01:45.412872075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:01:45.41659056Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-25T00:01:47.072718755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:01:47.535065507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:01:47.539253958Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:01:47.609405632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:01:47.613203354Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:01:48.740615921Z caller=checkpoint.go:378 msg="writing series" size="468 kB" level=debug ts=2026-04-25T00:01:48.740645048Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=warn ts=2026-04-25T00:01:52.072906316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:52.074021695Z caller=checkpoint.go:378 msg="writing series" size="19 kB" ts=2026-04-25T00:01:54.836680894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:01:55.407964911Z caller=checkpoint.go:378 msg="writing series" size="444 kB" level=info ts=2026-04-25T00:01:55.525202439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:01:55.529831664Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=debug ts=2026-04-25T00:01:56.574199058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=warn ts=2026-04-25T00:01:57.072770916Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:01:58.741048644Z caller=checkpoint.go:378 msg="writing series" size="13 kB" level=debug ts=2026-04-25T00:02:01.248461005Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:02:02.072652757Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:02:02.07268807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:02:02.07270072Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:02:02.072697445Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:02:02.073818261Z caller=checkpoint.go:378 msg="writing series" size="6.6 kB" level=info ts=2026-04-25T00:02:02.122937961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:02:02.127316414Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" ts=2026-04-25T00:02:02.253449228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:45202" level=info ts=2026-04-25T00:02:03.093845398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:02:03.099124674Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:02:03.602291796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:02:03.605495192Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:02:05.001989099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:02:05.005260611Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-25T00:02:05.407234139Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=warn ts=2026-04-25T00:02:07.073410452Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:02:08.225373737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:02:08.228758121Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-25T00:02:08.741263912Z caller=checkpoint.go:378 msg="writing series" size="106 kB" level=info ts=2026-04-25T00:02:08.924581757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:02:08.931289703Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:02:09.66711126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:02:09.672434476Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:02:09.965161191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:02:09.972306718Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:02:12.073201778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:02:12.074307078Z caller=checkpoint.go:378 msg="writing series" size="163 kB" level=info ts=2026-04-25T00:02:14.389879236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:02:14.395398814Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:02:15.407144743Z caller=checkpoint.go:378 msg="writing series" size="64 kB" level=warn ts=2026-04-25T00:02:17.072965212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:02:18.741043008Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=info ts=2026-04-25T00:02:19.447662322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:02:19.454800754Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-25T00:02:21.068896071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:02:21.072356751Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:02:22.072803236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:02:22.073944351Z caller=checkpoint.go:378 msg="writing series" size="189 kB" level=info ts=2026-04-25T00:02:23.418373335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:02:23.42661049Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" ts=2026-04-25T00:02:24.838633561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:02:25.407121596Z caller=checkpoint.go:378 msg="writing series" size="518 kB" level=debug ts=2026-04-25T00:02:25.407143185Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=11 level=warn ts=2026-04-25T00:02:27.073356438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:02:28.020001401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:02:28.024287107Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:02:28.740546761Z caller=checkpoint.go:378 msg="writing series" size="609 kB" level=debug ts=2026-04-25T00:02:30.336061599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:02:30.66385065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:02:30.667779089Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:02:31.868800846Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:02:31.869236084Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:02:32.073139604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:02:32.073171651Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:02:32.073180406Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:02:32.07325195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:02:32.075233054Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB" level=debug ts=2026-04-25T00:02:32.075246573Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=2 level=info ts=2026-04-25T00:02:34.157911934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:02:34.162990795Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-25T00:02:35.408261052Z caller=checkpoint.go:378 msg="writing series" size="816 kB" level=info ts=2026-04-25T00:02:35.858822461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:02:35.86284549Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-25T00:02:37.072700387Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:02:38.741217349Z caller=checkpoint.go:378 msg="writing series" size="61 kB" level=info ts=2026-04-25T00:02:39.149440775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:02:39.153462976Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:02:42.072960192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:02:42.074032822Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" ts=2026-04-25T00:02:42.096927497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:48380" level=debug ts=2026-04-25T00:02:43.511186142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-25T00:02:43.513061043Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-04-25T00:02:44.891592213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:02:44.896976813Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-25T00:02:45.407137151Z caller=checkpoint.go:378 msg="writing series" size="16 kB" level=info ts=2026-04-25T00:02:46.039993535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:02:46.046407245Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:02:47.072465922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:02:48.741052485Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB" level=info ts=2026-04-25T00:02:50.358239104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:02:50.365290356Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:02:50.766391472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:02:50.769931503Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-25T00:02:52.073181803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:02:52.074273713Z caller=checkpoint.go:378 msg="writing series" size="17 kB" level=info ts=2026-04-25T00:02:53.010545427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:02:53.013962892Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" ts=2026-04-25T00:02:53.443383184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:55674" level=info ts=2026-04-25T00:02:53.716671662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:02:53.721457745Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" ts=2026-04-25T00:02:54.841103692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=debug ts=2026-04-25T00:02:55.407761969Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB" level=warn ts=2026-04-25T00:02:57.073036646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:02:57.338908748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:02:57.344452011Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=debug ts=2026-04-25T00:02:57.64694707Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:02:57.812292802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:02:57.816288189Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:02:58.7417538Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB" level=debug ts=2026-04-25T00:02:58.741777585Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=8 level=info ts=2026-04-25T00:02:59.906925753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:02:59.910206215Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:03:00.584944575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:03:00.589717525Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-25T00:03:02.073129098Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:03:02.073160731Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:03:02.073169869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:03:02.073223273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:03:02.079654755Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016 level=info ts=2026-04-25T00:03:02.080800266Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007269846s ts=2026-04-25T00:03:02.258133197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:53488" level=info ts=2026-04-25T00:03:04.394617126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:03:04.402149037Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:03:07.072923965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:03:09.812475315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:03:09.819235128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:03:09.81925401Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:03:09.819284421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:03:09.825380988Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:03:12.073434679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:03:13.136785013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:03:13.14029703Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:03:15.407628104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:03:15.41171182Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:03:16.512652847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:03:16.519348055Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:03:16.51936735Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:03:16.519391832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:03:16.525576951Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-25T00:03:17.073155435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:03:19.759055168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:03:19.763018189Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:03:22.073172823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:03:22.581558641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:03:22.585222995Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:03:22.585244802Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:03:22.585265687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:03:22.588111494Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:03:23.772446623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:03:23.779020888Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" ts=2026-04-25T00:03:24.84351256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=warn ts=2026-04-25T00:03:27.072886017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:03:29.386443816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-04-25T00:03:31.142588716Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:03:31.46445349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:03:31.468491727Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:03:31.868838405Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:03:31.869240603Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:03:32.073192493Z caller=checkpoint.go:612 msg="starting checkpoint" level=info ts=2026-04-25T00:03:32.073204601Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:03:32.073231621Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:03:32.073240468Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:03:32.073251197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:03:32.073413481Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017 level=debug ts=2026-04-25T00:03:32.074919072Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB" level=debug ts=2026-04-25T00:03:32.074930434Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.3 MB" series=1 level=info ts=2026-04-25T00:03:32.085362754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:03:32.088791416Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" ts=2026-04-25T00:03:32.261219838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41874" level=info ts=2026-04-25T00:03:34.94218643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:03:34.949093374Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-25T00:03:35.407131995Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB" level=info ts=2026-04-25T00:03:35.780282971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:03:35.783562836Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-25T00:03:37.072851054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:03:38.223649518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:03:38.228582858Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:03:38.228602576Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:03:38.228624591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:03:38.23283407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:03:38.232880249Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:03:38.235629795Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-25T00:03:38.741088776Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB" level=warn ts=2026-04-25T00:03:42.07277417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:03:42.07387718Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=info ts=2026-04-25T00:03:43.701493013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:03:43.70730475Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:03:44.333288433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:03:44.336744107Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:03:44.47122065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:03:44.477477514Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:03:44.477497001Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:03:44.477522177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:03:44.483567487Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:03:45.407114853Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=warn ts=2026-04-25T00:03:47.072423759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:03:47.60837015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:03:47.611717602Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=debug ts=2026-04-25T00:03:48.741165047Z caller=checkpoint.go:378 msg="writing series" size="283 kB" level=info ts=2026-04-25T00:03:49.07137801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:03:49.076241313Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-25T00:03:49.368783284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:03:49.376711269Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:03:49.774370064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:03:49.779194307Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:03:50.92428625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:03:50.927853771Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:03:50.927878385Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:03:50.927906335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:03:50.930869749Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:03:51.180664864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:03:51.183834386Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-25T00:03:51.468704987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:03:51.472568242Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:03:51.472587752Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:03:51.472609595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:03:51.475813948Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:03:52.049164933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:03:52.055943879Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:03:52.073227162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:03:52.074303021Z caller=checkpoint.go:378 msg="writing series" size="16 kB" ts=2026-04-25T00:03:53.449179442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:54752" level=info ts=2026-04-25T00:03:54.529650513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:03:54.536440843Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" ts=2026-04-25T00:03:54.844563618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:03:55.407969411Z caller=checkpoint.go:378 msg="writing series" size="25 kB" level=warn ts=2026-04-25T00:03:57.073437391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:03:58.741396353Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB" level=info ts=2026-04-25T00:04:00.992262183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:04:00.995921302Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:04:02.07325562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:04:02.073283264Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:04:02.07329262Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:04:02.073311662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:04:02.074434919Z caller=checkpoint.go:378 msg="writing series" size="202 kB" level=info ts=2026-04-25T00:04:02.419586219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:04:02.423727081Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:04:04.331089736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:04:04.619344286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:04:04.62622539Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-25T00:04:05.408566938Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB" level=debug ts=2026-04-25T00:04:05.408589412Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.1 MB" series=10 level=info ts=2026-04-25T00:04:06.816834424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:04:06.820959971Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:04:06.820981056Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:06.821004129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:04:06.823888117Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:04:06.911608216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:04:06.914588203Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:04:07.073410439Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:04:08.741320157Z caller=checkpoint.go:378 msg="writing series" size="186 kB" level=info ts=2026-04-25T00:04:09.109435004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:04:09.112868284Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:04:09.925371286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:04:09.92871346Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:04:09.928731657Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:09.928752303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:04:09.931453468Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=warn ts=2026-04-25T00:04:12.07344731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:04:12.074580251Z caller=checkpoint.go:378 msg="writing series" size="196 kB" ts=2026-04-25T00:04:12.104350709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53550" level=info ts=2026-04-25T00:04:12.275896029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:04:12.279484158Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=error ts=2026-04-25T00:04:12.27950975Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:12.279536905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:04:12.283009531Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:04:14.657101415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:04:14.661797167Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:04:14.820445672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:04:14.823835906Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:04:14.823858579Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:14.823888011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:04:14.826845478Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-25T00:04:15.407951427Z caller=checkpoint.go:378 msg="writing series" size="45 kB" level=info ts=2026-04-25T00:04:16.249100973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:04:16.256260011Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:04:17.072893813Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:04:18.740977016Z caller=checkpoint.go:378 msg="writing series" size="496 kB" level=info ts=2026-04-25T00:04:19.129002321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:04:19.132431452Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-25T00:04:20.518142334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=warn ts=2026-04-25T00:04:22.073173178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:04:22.090053769Z caller=checkpoint.go:378 msg="writing series" size="13 MB" level=debug ts=2026-04-25T00:04:22.09008095Z caller=checkpoint.go:390 msg="flushing series" totalSize="14 MB" series=5 level=info ts=2026-04-25T00:04:22.933339787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:04:22.936740606Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" ts=2026-04-25T00:04:23.452549549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44250" level=info ts=2026-04-25T00:04:23.70094875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:04:23.704316773Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" ts=2026-04-25T00:04:24.846223828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:04:25.407767574Z caller=checkpoint.go:378 msg="writing series" size="63 kB" level=info ts=2026-04-25T00:04:26.883688863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:04:26.887695505Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-25T00:04:27.073250121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:04:27.549751352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:04:27.553223413Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-25T00:04:28.740802435Z caller=checkpoint.go:378 msg="writing series" size="97 kB" level=info ts=2026-04-25T00:04:29.002652133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:04:29.006352477Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:04:29.006372825Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:29.006393873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:04:29.009183325Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:04:29.381309264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:04:29.385015772Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-25T00:04:31.868386425Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:04:31.8694543Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-25T00:04:32.072795706Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:04:32.072828842Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:04:32.072837586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:04:32.07284954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:04:32.073955316Z caller=checkpoint.go:378 msg="writing series" size="51 kB" level=info ts=2026-04-25T00:04:32.88242775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:04:32.886429214Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:04:32.886447538Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:32.8864688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:04:32.889350349Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:04:34.817004689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:04:34.823555975Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:04:34.823574765Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:34.823599892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:04:34.829924458Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-25T00:04:35.407842477Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB" level=info ts=2026-04-25T00:04:35.717732078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:04:35.723431026Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:04:35.723450648Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:35.723473643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:04:35.728812503Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-25T00:04:36.978877042Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:04:37.073335661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:04:38.096677829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="843 kB" avg_comp="843 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:04:38.101048312Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a875a5%3A19dc1d771e0%3Ad970f4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=error ts=2026-04-25T00:04:38.101067319Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:38.101088958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:04:38.103940207Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=debug ts=2026-04-25T00:04:38.740558445Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB" level=info ts=2026-04-25T00:04:39.461651776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:04:39.46507764Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-25T00:04:42.073076729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:04:42.074109471Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=info ts=2026-04-25T00:04:42.691622607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:04:42.694938182Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:04:43.898340875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:04:43.90156135Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:04:43.90158224Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:43.901603746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:04:43.904372503Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-25T00:04:45.407651967Z caller=checkpoint.go:378 msg="writing series" size="122 kB" level=info ts=2026-04-25T00:04:45.519004852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:04:45.525453138Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:04:47.073512997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:04:47.871685537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:04:47.877026373Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:04:48.69120857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:04:48.69487875Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=error ts=2026-04-25T00:04:48.694895656Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:48.694920895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:04:48.697691689Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=debug ts=2026-04-25T00:04:48.740884171Z caller=checkpoint.go:378 msg="writing series" size="246 kB" level=info ts=2026-04-25T00:04:49.729897282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:04:49.7333142Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=info ts=2026-04-25T00:04:50.338962292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:04:50.342552426Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:04:50.342579898Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:04:50.342609016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:04:50.345450509Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:04:50.799749468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:04:50.803491673Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:04:50.907016312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:04:50.914007068Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:04:52.073010854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:04:52.074143934Z caller=checkpoint.go:378 msg="writing series" size="220 kB" level=info ts=2026-04-25T00:04:52.887008993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:04:52.891919609Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:04:53.388114205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:04:53.393431719Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" ts=2026-04-25T00:04:54.847727988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:04:55.407894505Z caller=checkpoint.go:378 msg="writing series" size="106 kB" level=info ts=2026-04-25T00:04:57.044556083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:04:57.048039997Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:04:57.073362903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:04:57.391866889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:04:57.395426408Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-25T00:04:58.740814778Z caller=checkpoint.go:378 msg="writing series" size="621 kB" level=debug ts=2026-04-25T00:04:58.740837925Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=11 level=info ts=2026-04-25T00:04:58.903206065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:04:58.906863182Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:05:00.229531991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:05:00.234583723Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:05:02.072707347Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:05:02.07273388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:05:02.07274262Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:05:02.072761251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:05:02.07408848Z caller=checkpoint.go:378 msg="writing series" size="529 kB" level=info ts=2026-04-25T00:05:03.57077356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:05:03.575556953Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-25T00:05:04.329498525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-25T00:05:04.623116394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:05:04.6307315Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-25T00:05:05.407305619Z caller=checkpoint.go:378 msg="writing series" size="415 kB" level=info ts=2026-04-25T00:05:06.640937553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:05:06.644224104Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-25T00:05:07.072862823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:05:07.315510273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:05:07.322679572Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-25T00:05:07.939266002Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:05:07.939336629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="852 kB" avg_uncomp="852 kB" idle=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:05:07.942738597Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:05:08.158292976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:05:08.161618381Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=debug ts=2026-04-25T00:05:08.740648316Z caller=checkpoint.go:378 msg="writing series" size="305 kB" level=debug ts=2026-04-25T00:05:08.740673366Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3 level=info ts=2026-04-25T00:05:09.171031842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:05:09.174671493Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:05:09.422938337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:05:09.429508534Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:05:11.720089123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:05:11.724027877Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-25T00:05:12.073498257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:05:12.074548377Z caller=checkpoint.go:378 msg="writing series" size="41 kB" ts=2026-04-25T00:05:12.108240419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55944" level=debug ts=2026-04-25T00:05:15.407584856Z caller=checkpoint.go:378 msg="writing series" size="109 kB" level=warn ts=2026-04-25T00:05:17.072796268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:05:18.741079573Z caller=checkpoint.go:378 msg="writing series" size="154 kB" level=info ts=2026-04-25T00:05:21.200602211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:05:21.204534954Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=warn ts=2026-04-25T00:05:22.072752988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:05:22.073877331Z caller=checkpoint.go:378 msg="writing series" size="221 kB" level=info ts=2026-04-25T00:05:22.512559335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:05:22.518364361Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:05:24.367916429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:05:24.371398743Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" ts=2026-04-25T00:05:24.850028176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:05:25.408107312Z caller=checkpoint.go:378 msg="writing series" size="125 kB" level=info ts=2026-04-25T00:05:25.456145561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:05:25.459414615Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=warn ts=2026-04-25T00:05:27.073131498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:05:28.740548387Z caller=checkpoint.go:378 msg="writing series" size="128 kB" level=info ts=2026-04-25T00:05:31.464926543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:05:31.468132192Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:05:31.68673369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:05:31.689981358Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:05:31.868777562Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:05:31.869148786Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:05:32.073370535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:05:32.073403271Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:05:32.073411648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:05:32.073423263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:05:32.074492488Z caller=checkpoint.go:378 msg="writing series" size="57 kB" ts=2026-04-25T00:05:32.270549845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:33780" level=info ts=2026-04-25T00:05:34.397750296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:05:34.401678217Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:05:35.407914535Z caller=checkpoint.go:378 msg="writing series" size="19 kB" level=debug ts=2026-04-25T00:05:36.297940614Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:05:37.072966912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:05:37.799568915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:05:37.80329259Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-25T00:05:38.743391057Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB" level=debug ts=2026-04-25T00:05:38.74341589Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.3 MB" series=9 level=info ts=2026-04-25T00:05:39.487068625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:05:39.490408557Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:05:41.468149145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:05:41.471431772Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:05:42.072845639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:05:42.075523351Z caller=checkpoint.go:378 msg="writing series" size="3.4 MB" level=debug ts=2026-04-25T00:05:42.07553865Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.4 MB" series=1 ts=2026-04-25T00:05:42.109404578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58994" level=info ts=2026-04-25T00:05:44.42362394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:05:44.427460887Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=debug ts=2026-04-25T00:05:45.407797804Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB" level=warn ts=2026-04-25T00:05:47.072478251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:05:47.401991493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:05:47.405438613Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:05:47.465653073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:05:47.470612407Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:05:48.241045568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:05:48.244773947Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:05:48.741204927Z caller=checkpoint.go:378 msg="writing series" size="23 kB" level=debug ts=2026-04-25T00:05:48.74123Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=info ts=2026-04-25T00:05:49.316526927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:05:49.323266549Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:05:52.073473242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:05:52.074521575Z caller=checkpoint.go:378 msg="writing series" size="939 B" level=info ts=2026-04-25T00:05:53.784438345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:05:53.792982915Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:05:54.456180136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:05:54.459776264Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" ts=2026-04-25T00:05:54.852873701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=debug ts=2026-04-25T00:05:55.407602264Z caller=checkpoint.go:378 msg="writing series" size="817 B" level=info ts=2026-04-25T00:05:56.669264404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:05:56.67699441Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:05:57.073203962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:05:58.741223743Z caller=checkpoint.go:378 msg="writing series" size="218 kB" level=debug ts=2026-04-25T00:05:59.453468205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-25T00:06:00.521030545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:06:00.528476579Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:06:02.073259485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:06:02.073288856Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:06:02.073296643Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:06:02.073302345Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:02.075293622Z caller=checkpoint.go:378 msg="writing series" size="627 kB" level=info ts=2026-04-25T00:06:02.127515536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:06:02.132066502Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:06:05.4078939Z caller=checkpoint.go:378 msg="writing series" size="609 kB" level=debug ts=2026-04-25T00:06:05.407918514Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5 level=info ts=2026-04-25T00:06:06.216092516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:06:06.220972629Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:06:06.23102368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:06:06.233887856Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-25T00:06:07.073061474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:08.740847012Z caller=checkpoint.go:378 msg="writing series" size="248 kB" level=debug ts=2026-04-25T00:06:11.943383664Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:06:11.94339136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:06:11.947231718Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=warn ts=2026-04-25T00:06:12.072717271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:12.073933152Z caller=checkpoint.go:378 msg="writing series" size="411 kB" ts=2026-04-25T00:06:12.111066594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60584" level=info ts=2026-04-25T00:06:15.232862923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:06:15.239495666Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-25T00:06:15.407319196Z caller=checkpoint.go:378 msg="writing series" size="528 B" level=warn ts=2026-04-25T00:06:17.073037332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:18.740616915Z caller=checkpoint.go:378 msg="writing series" size="130 kB" level=info ts=2026-04-25T00:06:21.379912183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:06:21.38365239Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:06:21.79766331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:06:21.803242356Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-25T00:06:22.07343763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:22.074568852Z caller=checkpoint.go:378 msg="writing series" size="410 kB" level=debug ts=2026-04-25T00:06:22.074587152Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5 level=info ts=2026-04-25T00:06:23.431151137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:06:23.434856363Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:06:24.11052349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:06:24.113873365Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" ts=2026-04-25T00:06:24.855314169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:06:25.407486673Z caller=checkpoint.go:378 msg="writing series" size="36 kB" level=info ts=2026-04-25T00:06:25.695423579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:06:25.698723467Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:06:27.072898271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:06:28.442594814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:06:28.446065319Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=debug ts=2026-04-25T00:06:28.740473541Z caller=checkpoint.go:378 msg="writing series" size="85 kB" level=info ts=2026-04-25T00:06:29.438377019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:06:29.44210235Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:06:30.793089655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:06:30.79687008Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:06:31.868611739Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:06:31.868944697Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:06:32.072380919Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:06:32.072409142Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:06:32.072417781Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:06:32.072441971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:32.074685962Z caller=checkpoint.go:378 msg="writing series" size="149 kB" level=info ts=2026-04-25T00:06:32.683292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:06:32.691228813Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-25T00:06:32.889010676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:06:32.892216218Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:06:35.117595831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:06:35.124583285Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-25T00:06:35.408245653Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB" level=debug ts=2026-04-25T00:06:35.40827481Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4 level=warn ts=2026-04-25T00:06:37.073427754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:06:38.168912085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:06:38.173421505Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=debug ts=2026-04-25T00:06:38.740865433Z caller=checkpoint.go:378 msg="writing series" size="116 kB" level=info ts=2026-04-25T00:06:40.412932514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:06:40.419165294Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=debug ts=2026-04-25T00:06:41.175043691Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:06:41.779576624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:06:41.783257595Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-25T00:06:42.072778579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:42.099833155Z caller=checkpoint.go:378 msg="writing series" size="49 MB" level=debug ts=2026-04-25T00:06:42.099858417Z caller=checkpoint.go:390 msg="flushing series" totalSize="49 MB" series=2 ts=2026-04-25T00:06:42.113149391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:59182" level=info ts=2026-04-25T00:06:42.667753649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:06:42.671129766Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-25T00:06:45.408078128Z caller=checkpoint.go:378 msg="writing series" size="676 kB" level=warn ts=2026-04-25T00:06:47.073554019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:47.609898319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-25T00:06:47.611396994Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-04-25T00:06:48.740605046Z caller=checkpoint.go:378 msg="writing series" size="468 kB" level=debug ts=2026-04-25T00:06:48.740628839Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=info ts=2026-04-25T00:06:50.914060158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:06:50.921478631Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:06:51.402781706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:06:51.406455865Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:06:51.445590706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:06:51.44888885Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=warn ts=2026-04-25T00:06:52.072722012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:52.073848327Z caller=checkpoint.go:378 msg="writing series" size="20 kB" ts=2026-04-25T00:06:54.8580141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-25T00:06:55.187378709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:06:55.191158645Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:06:55.197357293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:06:55.203593658Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-25T00:06:55.407317652Z caller=checkpoint.go:378 msg="writing series" size="575 kB" level=warn ts=2026-04-25T00:06:57.072684314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:06:58.740858946Z caller=checkpoint.go:378 msg="writing series" size="13 kB" level=info ts=2026-04-25T00:07:00.6834698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:07:00.687091696Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:07:02.072680341Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:07:02.072711444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:07:02.072720861Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:07:02.072739708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:07:02.073786154Z caller=checkpoint.go:378 msg="writing series" size="6.6 kB" level=info ts=2026-04-25T00:07:04.027464555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:07:04.031303329Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-25T00:07:05.407688105Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=info ts=2026-04-25T00:07:06.634263129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:07:06.639827269Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:07:07.073028375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:07:07.909368214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:07:07.914693122Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-25T00:07:08.740914117Z caller=checkpoint.go:378 msg="writing series" size="138 kB" level=info ts=2026-04-25T00:07:10.049256002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:07:10.053880587Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:07:10.128197694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:07:10.131628072Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=warn ts=2026-04-25T00:07:12.073070339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:07:12.07423697Z caller=checkpoint.go:378 msg="writing series" size="158 kB" level=debug ts=2026-04-25T00:07:13.738054053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:07:13.929732809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:07:13.93351379Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:07:15.117472736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:07:15.120886165Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-25T00:07:15.407244747Z caller=checkpoint.go:378 msg="writing series" size="65 kB" level=info ts=2026-04-25T00:07:15.869085262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:07:15.872465748Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=warn ts=2026-04-25T00:07:17.073316409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:07:18.740350526Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=info ts=2026-04-25T00:07:21.827891649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:07:21.836708614Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-25T00:07:22.072718162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:07:22.07384262Z caller=checkpoint.go:378 msg="writing series" size="190 kB" level=debug ts=2026-04-25T00:07:22.073858421Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10 level=info ts=2026-04-25T00:07:22.394246006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:07:22.397602555Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" ts=2026-04-25T00:07:24.861123337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:07:25.407537292Z caller=checkpoint.go:378 msg="writing series" size="376 kB" level=info ts=2026-04-25T00:07:25.911919032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:07:25.917128884Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-25T00:07:27.072455704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:07:28.74124484Z caller=checkpoint.go:378 msg="writing series" size="387 kB" level=info ts=2026-04-25T00:07:29.323612496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:07:29.330605112Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-25T00:07:31.868646909Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:07:31.868649747Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:07:32.072901019Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:07:32.072931761Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:07:32.072944926Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:07:32.072964881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:07:32.074904878Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB" level=debug ts=2026-04-25T00:07:32.074923368Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=3 level=debug ts=2026-04-25T00:07:35.407541625Z caller=checkpoint.go:378 msg="writing series" size="816 kB" level=info ts=2026-04-25T00:07:35.68420233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:07:35.688457983Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-25T00:07:36.728521114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:07:36.732759394Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:07:37.073262917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:07:37.20254284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:07:37.207653659Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:07:38.741385531Z caller=checkpoint.go:378 msg="writing series" size="61 kB" level=debug ts=2026-04-25T00:07:38.800890141Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:07:39.651166642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:07:39.655076116Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:07:39.735795737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:07:39.740187702Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=debug ts=2026-04-25T00:07:40.908090959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=warn ts=2026-04-25T00:07:42.073254388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:07:42.074354693Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=info ts=2026-04-25T00:07:44.969553234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:07:44.976389577Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-25T00:07:45.407951326Z caller=checkpoint.go:378 msg="writing series" size="16 kB" level=info ts=2026-04-25T00:07:46.886895778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:07:46.890416478Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-25T00:07:47.073052634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:07:47.619183906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:07:47.626527333Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-25T00:07:48.740315057Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB" level=info ts=2026-04-25T00:07:49.120997428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:07:49.124727714Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:07:49.24929554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:07:49.252867053Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:07:49.67092019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:07:49.677976966Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:07:51.942172224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:07:51.946110813Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=warn ts=2026-04-25T00:07:52.073398774Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:07:52.074515262Z caller=checkpoint.go:378 msg="writing series" size="17 kB" level=info ts=2026-04-25T00:07:53.311590425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:07:53.316069091Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" ts=2026-04-25T00:07:54.86423215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:07:55.407658554Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB" level=warn ts=2026-04-25T00:07:57.072892646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:07:58.644440261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:07:58.648686419Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-25T00:07:58.742014511Z caller=checkpoint.go:378 msg="writing series" size="1.9 MB" level=debug ts=2026-04-25T00:07:58.742039079Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=8 level=info ts=2026-04-25T00:08:01.775745602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:08:01.779846204Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:08:02.07287815Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:08:02.072888693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:08:02.07291361Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:08:02.072923169Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-25T00:08:02.081506833Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017 level=info ts=2026-04-25T00:08:02.082657755Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009085495s level=debug ts=2026-04-25T00:08:04.850451036Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:08:05.973052468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:08:05.977404732Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=warn ts=2026-04-25T00:08:07.073366626Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:08:07.857464316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:08:07.862308708Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:08:09.867581021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:08:09.875854372Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-25T00:08:12.073473385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:08:12.867741071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:08:12.873544406Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:08:13.668839366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:08:13.676432012Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:08:17.073012056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:08:22.050905631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:08:22.054878936Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-25T00:08:22.073466753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:08:23.470876709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:40110" ts=2026-04-25T00:08:24.867572642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=debug ts=2026-04-25T00:08:25.398991604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=warn ts=2026-04-25T00:08:27.072689329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:08:31.161605718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:08:31.169877012Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:08:31.347224908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:08:31.351495964Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:08:31.363480104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:08:31.366957024Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:08:31.868637641Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:08:31.868639272Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=debug ts=2026-04-25T00:08:31.967879957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:08:31.991916043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:08:31.998941319Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:08:32.073188394Z caller=checkpoint.go:612 msg="starting checkpoint" level=warn ts=2026-04-25T00:08:32.073206379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:08:32.073210862Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:08:32.073233055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:08:32.073245711Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-25T00:08:32.073467884Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018 level=debug ts=2026-04-25T00:08:32.075266295Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB" level=debug ts=2026-04-25T00:08:32.07527801Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=1 ts=2026-04-25T00:08:32.284622331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:50504" level=info ts=2026-04-25T00:08:32.875587092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:08:32.879471742Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:08:33.29970135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:08:33.303589844Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=debug ts=2026-04-25T00:08:35.407387791Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB" level=info ts=2026-04-25T00:08:35.74725913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:08:35.751524822Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:08:35.766732953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:08:35.77047868Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:08:35.893931941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:08:35.897562867Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:08:36.852555569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:08:36.85801246Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:08:37.072705751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:08:38.701379525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:08:38.70528254Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-25T00:08:38.741399865Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB" level=info ts=2026-04-25T00:08:41.507514015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:08:41.512709269Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-25T00:08:42.073179219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:08:42.074276722Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=info ts=2026-04-25T00:08:43.89193696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:08:43.897430954Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=debug ts=2026-04-25T00:08:45.407840368Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=info ts=2026-04-25T00:08:46.413015559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:08:46.416985625Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-25T00:08:47.073057389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:08:48.740916306Z caller=checkpoint.go:378 msg="writing series" size="322 kB" level=warn ts=2026-04-25T00:08:52.073228782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:08:52.074293127Z caller=checkpoint.go:378 msg="writing series" size="17 kB" ts=2026-04-25T00:08:54.870419061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:08:55.407830864Z caller=checkpoint.go:378 msg="writing series" size="25 kB" level=warn ts=2026-04-25T00:08:57.072807221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:08:58.741235595Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB" level=debug ts=2026-04-25T00:08:59.63275493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:08:59.63284031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="363 kB" avg_comp="363 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" idle=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:08:59.639942952Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=info ts=2026-04-25T00:09:01.690895234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:09:01.695944649Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:09:02.073004735Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:09:02.07303468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:09:02.073043461Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:09:02.073063746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:09:02.07416733Z caller=checkpoint.go:378 msg="writing series" size="202 kB" level=info ts=2026-04-25T00:09:02.91839176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:09:02.923700556Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:09:03.260596141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:09:03.264632028Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:09:05.4093144Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB" level=debug ts=2026-04-25T00:09:05.409381967Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.1 MB" series=10 level=warn ts=2026-04-25T00:09:07.073235274Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:09:08.29788071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:09:08.304923334Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:09:08.612962197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:09:08.619718956Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-25T00:09:08.741294173Z caller=checkpoint.go:378 msg="writing series" size="189 kB" level=info ts=2026-04-25T00:09:09.686590406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:09:09.690913777Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-25T00:09:09.81116956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-25T00:09:11.367374375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:09:11.371096848Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:09:12.073188716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:09:12.074382245Z caller=checkpoint.go:378 msg="writing series" size="196 kB" ts=2026-04-25T00:09:12.127236553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:54184" level=debug ts=2026-04-25T00:09:15.407948743Z caller=checkpoint.go:378 msg="writing series" size="46 kB" level=info ts=2026-04-25T00:09:16.554378874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:09:16.562650414Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:09:17.072873226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:09:17.221163526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:09:17.225830284Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=debug ts=2026-04-25T00:09:18.741078537Z caller=checkpoint.go:378 msg="writing series" size="496 kB" level=info ts=2026-04-25T00:09:19.563388602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:09:19.570011369Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:09:19.835604496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:09:19.839154716Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-25T00:09:20.203452739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:09:20.207295189Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:09:20.349418354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:09:20.35318825Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:09:20.753224143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:09:20.756797966Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:09:22.073152419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:09:22.085137997Z caller=checkpoint.go:378 msg="writing series" size="13 MB" level=debug ts=2026-04-25T00:09:22.085158873Z caller=checkpoint.go:390 msg="flushing series" totalSize="14 MB" series=5 ts=2026-04-25T00:09:23.476568057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:51130" ts=2026-04-25T00:09:24.872057773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:09:25.407436482Z caller=checkpoint.go:378 msg="writing series" size="63 kB" level=debug ts=2026-04-25T00:09:26.194190269Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:09:26.459835685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:09:26.463535432Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-25T00:09:27.072777952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:09:28.74060733Z caller=checkpoint.go:378 msg="writing series" size="97 kB" level=info ts=2026-04-25T00:09:30.692195402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:09:30.696686503Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:09:31.869137822Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:09:31.86961066Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:09:32.072880051Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:09:32.072905318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:09:32.072921372Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:09:32.072934274Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:09:32.07400936Z caller=checkpoint.go:378 msg="writing series" size="51 kB" level=info ts=2026-04-25T00:09:32.671536267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:09:32.678302851Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:09:32.74345749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:09:32.747155773Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-25T00:09:35.407149822Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB" level=info ts=2026-04-25T00:09:35.621835759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:09:35.625850578Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:09:37.072702719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:09:38.74106426Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB" level=info ts=2026-04-25T00:09:39.877094279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:09:39.882203456Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-25T00:09:42.073163194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:09:42.07423925Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=info ts=2026-04-25T00:09:42.173557614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:09:42.178546683Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=info ts=2026-04-25T00:09:43.071542508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:09:43.075024262Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=info ts=2026-04-25T00:09:44.637839384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:09:44.64137741Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:09:45.40738018Z caller=checkpoint.go:378 msg="writing series" size="122 kB" level=info ts=2026-04-25T00:09:46.415299076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:09:46.418958382Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-25T00:09:47.07277903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:09:48.741526082Z caller=checkpoint.go:378 msg="writing series" size="246 kB" level=warn ts=2026-04-25T00:09:52.073454566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:09:52.0745623Z caller=checkpoint.go:378 msg="writing series" size="191 kB" level=debug ts=2026-04-25T00:09:53.259602296Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:09:53.495480403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:09:53.50343623Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:09:53.50346097Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:09:53.503501079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:09:53.510481969Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-25T00:09:54.746093778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:09:54.7503762Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" ts=2026-04-25T00:09:54.874123555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:09:55.407543479Z caller=checkpoint.go:378 msg="writing series" size="109 kB" level=warn ts=2026-04-25T00:09:57.073516745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:09:58.29753243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:09:58.302605908Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:09:58.302632495Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:09:58.302659198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:09:58.307109584Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=debug ts=2026-04-25T00:09:58.355960687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-04-25T00:09:58.741380322Z caller=checkpoint.go:378 msg="writing series" size="492 kB" level=debug ts=2026-04-25T00:09:58.74140596Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=11 level=info ts=2026-04-25T00:10:02.073366832Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:10:02.07340275Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=warn ts=2026-04-25T00:10:02.073402632Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:10:02.073414232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:10:02.074837566Z caller=checkpoint.go:378 msg="writing series" size="529 kB" ts=2026-04-25T00:10:02.291531756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55656" level=info ts=2026-04-25T00:10:03.469307791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:10:03.479659295Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:10:03.779392541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:10:03.786337589Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-25T00:10:05.091197739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:10:05.098421927Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:10:05.098447755Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:05.098478476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:10:05.105214797Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:10:05.407768066Z caller=checkpoint.go:378 msg="writing series" size="415 kB" level=info ts=2026-04-25T00:10:05.782812086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:10:05.786588348Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=warn ts=2026-04-25T00:10:07.073396893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:10:07.496164992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:10:07.504084073Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:10:07.50411159Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:07.504141708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:10:07.510876405Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:10:08.313931897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:10:08.317449219Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:10:08.349534226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:10:08.352930387Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:10:08.741502947Z caller=checkpoint.go:378 msg="writing series" size="367 kB" level=debug ts=2026-04-25T00:10:08.741526461Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3 level=info ts=2026-04-25T00:10:10.718711379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:10:10.722490884Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:10:10.722520558Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:10.722550496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:10:10.72572903Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-25T00:10:12.073380734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:10:12.074534811Z caller=checkpoint.go:378 msg="writing series" size="41 kB" ts=2026-04-25T00:10:12.131786264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:56878" level=info ts=2026-04-25T00:10:12.515546031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:10:12.518883383Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:10:13.893764949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:10:13.900597457Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-25T00:10:15.407062703Z caller=checkpoint.go:378 msg="writing series" size="109 kB" level=info ts=2026-04-25T00:10:15.495450146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:10:15.502718512Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:10:15.751154506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:10:15.754735968Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:10:15.754755267Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:15.754777045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:10:15.757558218Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:10:17.07331061Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:10:17.084380462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:10:17.087758502Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:10:17.238093943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:10:17.242007503Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=error ts=2026-04-25T00:10:17.242032782Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:17.242061497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:10:17.245097564Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:10:17.605646092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:10:17.610984092Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:10:18.096535526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:10:18.103559687Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:10:18.372381933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:10:18.375902165Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=debug ts=2026-04-25T00:10:18.741103944Z caller=checkpoint.go:378 msg="writing series" size="163 kB" level=info ts=2026-04-25T00:10:20.84860572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:10:20.854617065Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:10:21.563984009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:10:21.568252187Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:10:21.568277467Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:21.568302921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:10:21.57191954Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-25T00:10:22.073387915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:10:22.074570178Z caller=checkpoint.go:378 msg="writing series" size="221 kB" level=info ts=2026-04-25T00:10:22.186566321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:10:22.190390149Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" ts=2026-04-25T00:10:24.876385421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:10:25.407709877Z caller=checkpoint.go:378 msg="writing series" size="125 kB" level=debug ts=2026-04-25T00:10:26.1000772Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:10:27.072782227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:10:27.633675625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:10:27.638938637Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:10:27.687767556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:10:27.691512868Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=info ts=2026-04-25T00:10:28.012394731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:10:28.016999688Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=debug ts=2026-04-25T00:10:28.741223671Z caller=checkpoint.go:378 msg="writing series" size="154 kB" level=info ts=2026-04-25T00:10:31.868621945Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:10:31.868946972Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:10:32.073429602Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:10:32.07346041Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=warn ts=2026-04-25T00:10:32.073449529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:10:32.073470348Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:10:32.074599044Z caller=checkpoint.go:378 msg="writing series" size="60 kB" level=info ts=2026-04-25T00:10:32.319617183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:10:32.322970125Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-25T00:10:34.675596059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-25T00:10:34.677218867Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-04-25T00:10:34.681247407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:10:34.685109223Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:10:35.332975911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:10:35.336706677Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:10:35.336733559Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:35.336761155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:10:35.340136989Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:10:35.407380162Z caller=checkpoint.go:378 msg="writing series" size="19 kB" level=info ts=2026-04-25T00:10:36.648993698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:10:36.653865263Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-25T00:10:37.072670216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:10:38.745138114Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB" level=debug ts=2026-04-25T00:10:38.745164475Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.4 MB" series=9 level=info ts=2026-04-25T00:10:39.879447157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:10:39.886698906Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:10:42.073277629Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:10:42.078360367Z caller=checkpoint.go:378 msg="writing series" size="3.5 MB" level=debug ts=2026-04-25T00:10:42.078382198Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.5 MB" series=1 ts=2026-04-25T00:10:42.13350036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:59668" level=debug ts=2026-04-25T00:10:45.408074108Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB" level=info ts=2026-04-25T00:10:45.733583663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:10:45.738659454Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:10:46.818792272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:10:46.824426196Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-25T00:10:47.072517433Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:10:47.10764108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:10:47.114671668Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:10:48.740953138Z caller=checkpoint.go:378 msg="writing series" size="23 kB" level=debug ts=2026-04-25T00:10:48.740978818Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=info ts=2026-04-25T00:10:49.583849445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:10:49.591363209Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-25T00:10:52.073368444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:10:52.074467179Z caller=checkpoint.go:378 msg="writing series" size="939 B" ts=2026-04-25T00:10:54.878904407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-25T00:10:54.900384254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:10:54.904245347Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:10:55.10498313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:10:55.10875582Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-25T00:10:55.288384395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:10:55.292144118Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:10:55.402994133Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-25T00:10:55.407972987Z caller=checkpoint.go:378 msg="writing series" size="817 B" level=info ts=2026-04-25T00:10:56.481380056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:10:56.485144237Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-25T00:10:57.072506943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:10:58.128504751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:10:58.137216781Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:10:58.13724432Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:58.137274663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:10:58.143962859Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:10:58.214130303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:10:58.217893246Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:10:58.217920478Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:58.437366971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:10:58.441784281Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-25T00:10:58.740857303Z caller=checkpoint.go:378 msg="writing series" size="231 kB" level=info ts=2026-04-25T00:10:59.221341261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:10:59.225106025Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:10:59.225129874Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:10:59.225154235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:10:59.22815795Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:11:01.187020476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:11:01.191135602Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:11:01.554164111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:11:01.557758579Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:11:01.557781129Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:11:01.557804723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:11:01.560755534Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:11:02.073122297Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:11:02.073153477Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:11:02.073163133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:11:02.073160852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:11:02.074728545Z caller=checkpoint.go:378 msg="writing series" size="652 kB" level=info ts=2026-04-25T00:11:05.065592552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:11:05.069154179Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-25T00:11:05.408071313Z caller=checkpoint.go:378 msg="writing series" size="609 kB" level=debug ts=2026-04-25T00:11:05.408094852Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5 level=warn ts=2026-04-25T00:11:07.073218394Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:11:07.413382143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:11:07.419228862Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:11:07.419259773Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:11:07.419290142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:11:07.425212622Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-25T00:11:08.741366817Z caller=checkpoint.go:378 msg="writing series" size="336 kB" level=info ts=2026-04-25T00:11:08.979134898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:11:08.982710645Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:11:08.982732106Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:11:08.982754307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:11:08.985526798Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:11:10.667573936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:11:10.671423327Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=warn ts=2026-04-25T00:11:12.072537398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:11:12.074772722Z caller=checkpoint.go:378 msg="writing series" size="382 kB" ts=2026-04-25T00:11:12.13531405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35048" level=info ts=2026-04-25T00:11:12.92934124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:11:12.936225206Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:11:13.002523473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:11:13.006005166Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:11:13.888895573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:11:13.892387368Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-25T00:11:15.407892124Z caller=checkpoint.go:378 msg="writing series" size="528 B" level=info ts=2026-04-25T00:11:15.961777221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:11:15.965589571Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:11:15.965615748Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:11:15.965642658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:11:15.968784716Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:11:17.072822963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:11:18.74045911Z caller=checkpoint.go:378 msg="writing series" size="130 kB" level=info ts=2026-04-25T00:11:20.716361091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:11:20.720211401Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:11:20.87896013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:11:20.882428539Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:11:22.072944669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:11:22.074160422Z caller=checkpoint.go:378 msg="writing series" size="410 kB" level=debug ts=2026-04-25T00:11:22.074176553Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5 level=debug ts=2026-04-25T00:11:24.1316976Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:11:24.371496037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:11:24.376911281Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" ts=2026-04-25T00:11:24.881535365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:11:25.407645357Z caller=checkpoint.go:378 msg="writing series" size="36 kB" level=info ts=2026-04-25T00:11:25.805666621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:11:25.809872041Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=info ts=2026-04-25T00:11:26.118881819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}" level=error ts=2026-04-25T00:11:26.122639363Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dc1a875a5%3A19dc1a875a6%3Ab7bff5ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}" level=error ts=2026-04-25T00:11:26.122664823Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:11:26.122692591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:11:26.12654353Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-25T00:11:26.753252448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:11:26.757116982Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-25T00:11:27.073384484Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:11:27.767391193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:11:27.770843096Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:11:27.770865684Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:11:27.770887371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:11:27.773673873Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-25T00:11:28.740586606Z caller=checkpoint.go:378 msg="writing series" size="90 kB" level=info ts=2026-04-25T00:11:29.666674962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:11:29.671982525Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:11:31.541581682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:11:31.549461194Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:11:31.868461957Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:11:31.869527264Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=warn ts=2026-04-25T00:11:32.073080858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:11:32.073069039Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:11:32.073102213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:11:32.07311227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:11:32.074255668Z caller=checkpoint.go:378 msg="writing series" size="156 kB" ts=2026-04-25T00:11:32.298291557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:46228" level=info ts=2026-04-25T00:11:33.695496283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:11:33.705928232Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-25T00:11:35.408899492Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB" level=debug ts=2026-04-25T00:11:35.408927487Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=4 level=debug ts=2026-04-25T00:11:36.607103376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-25T00:11:36.727513623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:11:36.731187942Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:11:36.73121005Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:11:36.731232888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:11:36.734428108Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=warn ts=2026-04-25T00:11:37.073048802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:11:37.427005296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:11:37.432383449Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:11:38.284754203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:11:38.288725914Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=debug ts=2026-04-25T00:11:38.741000184Z caller=checkpoint.go:378 msg="writing series" size="116 kB" level=info ts=2026-04-25T00:11:39.766381321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:11:39.769900304Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:11:39.832023595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:11:39.839452618Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:11:40.958233885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:11:40.961854585Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:11:42.072813944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:11:42.110863284Z caller=checkpoint.go:378 msg="writing series" size="49 MB" level=debug ts=2026-04-25T00:11:42.110892614Z caller=checkpoint.go:390 msg="flushing series" totalSize="50 MB" series=2 level=info ts=2026-04-25T00:11:43.49399507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-25T00:11:43.497498702Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dc1a875a5%3A19dc1a8b9ec%3Af62ee4d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=error ts=2026-04-25T00:11:43.497519571Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:11:43.497542812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:11:43.500727351Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:11:45.104706108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:11:45.108464399Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:11:45.391802225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:11:45.395545207Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-25T00:11:45.407991654Z caller=checkpoint.go:378 msg="writing series" size="676 kB" level=info ts=2026-04-25T00:11:46.04169222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:11:46.045617436Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:11:46.53820898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:11:46.541867538Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-25T00:11:47.072461661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:11:48.741371799Z caller=checkpoint.go:378 msg="writing series" size="468 kB" level=debug ts=2026-04-25T00:11:48.741415622Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=info ts=2026-04-25T00:11:50.494738735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:11:50.498455013Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:11:50.75046652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:11:50.754305618Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:11:51.277289438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:11:51.281370505Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:11:51.80389291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:11:51.811874371Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:11:52.072897996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:11:52.073951357Z caller=checkpoint.go:378 msg="writing series" size="20 kB" level=info ts=2026-04-25T00:11:52.082089611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:11:52.089177095Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-25T00:11:52.781494577Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 ts=2026-04-25T00:11:53.488148896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:40784" level=info ts=2026-04-25T00:11:53.558850972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:11:53.562433963Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:11:54.112344144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:11:54.116551872Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" ts=2026-04-25T00:11:54.884764525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:11:55.408182092Z caller=checkpoint.go:378 msg="writing series" size="475 kB" level=warn ts=2026-04-25T00:11:57.0734567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:11:57.211825547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:11:57.215575174Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=debug ts=2026-04-25T00:11:58.741388658Z caller=checkpoint.go:378 msg="writing series" size="13 kB" level=info ts=2026-04-25T00:12:02.073130733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:12:02.073161791Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:12:02.073170942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:12:02.073178306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:02.074282224Z caller=checkpoint.go:378 msg="writing series" size="6.6 kB" level=info ts=2026-04-25T00:12:02.89339825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:12:02.899939688Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=debug ts=2026-04-25T00:12:05.407647953Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=warn ts=2026-04-25T00:12:07.073246146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:08.741417085Z caller=checkpoint.go:378 msg="writing series" size="169 kB" level=info ts=2026-04-25T00:12:10.382376417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:12:10.388520826Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-25T00:12:10.392574048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:12:10.396959622Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=warn ts=2026-04-25T00:12:12.073469021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:12.074552395Z caller=checkpoint.go:378 msg="writing series" size="158 kB" ts=2026-04-25T00:12:12.140571955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:51932" level=info ts=2026-04-25T00:12:12.41369863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:12:12.417838687Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:12:12.606013084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:12:12.609364004Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:12:14.278629757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-04-25T00:12:15.407574397Z caller=checkpoint.go:378 msg="writing series" size="66 kB" level=warn ts=2026-04-25T00:12:17.073417393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:12:17.516468423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:12:17.520086115Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:12:17.591319864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:12:17.599172694Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:12:18.741201956Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=warn ts=2026-04-25T00:12:22.072628969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:22.073790419Z caller=checkpoint.go:378 msg="writing series" size="191 kB" level=debug ts=2026-04-25T00:12:22.073808923Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10 ts=2026-04-25T00:12:23.491896298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:37214" level=info ts=2026-04-25T00:12:23.84590668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:12:23.8531029Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" ts=2026-04-25T00:12:24.887431695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:12:25.40776306Z caller=checkpoint.go:378 msg="writing series" size="422 kB" level=info ts=2026-04-25T00:12:25.961651395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:12:25.968905386Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-25T00:12:27.073561695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:12:27.346395001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:12:27.350281967Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-25T00:12:28.08863578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-25T00:12:28.741376916Z caller=checkpoint.go:378 msg="writing series" size="402 kB" level=info ts=2026-04-25T00:12:29.347009967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:12:29.35446581Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-25T00:12:31.868496772Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:12:31.869535027Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-25T00:12:32.072745585Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:12:32.072740415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:32.072786405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:12:32.072795399Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:12:32.074727517Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB" level=debug ts=2026-04-25T00:12:32.074740266Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=3 ts=2026-04-25T00:12:32.302886373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:44058" level=info ts=2026-04-25T00:12:34.715543477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:12:34.721167216Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-25T00:12:35.407611493Z caller=checkpoint.go:378 msg="writing series" size="816 kB" level=info ts=2026-04-25T00:12:36.352856017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:12:36.356651907Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-25T00:12:37.072883708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:12:37.527072494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:12:37.531279084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:12:38.019453975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:12:38.025387306Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-25T00:12:38.74124703Z caller=checkpoint.go:378 msg="writing series" size="61 kB" level=info ts=2026-04-25T00:12:41.819654803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:12:41.823585643Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:12:41.823609804Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:12:41.823636198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:12:41.82657599Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=warn ts=2026-04-25T00:12:42.072449767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:42.074608571Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" ts=2026-04-25T00:12:42.141815144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60204" level=info ts=2026-04-25T00:12:43.826940274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:12:43.83371646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:12:44.035450693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:12:44.03915882Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:12:44.082517277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:12:44.086584071Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:12:44.716179189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:12:44.719960457Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:12:45.407817268Z caller=checkpoint.go:378 msg="writing series" size="16 kB" level=info ts=2026-04-25T00:12:45.696798096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:12:45.700961724Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:12:45.784179393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:12:45.788363282Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-25T00:12:47.072719529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:48.740712016Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB" level=warn ts=2026-04-25T00:12:52.072597192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:52.073690872Z caller=checkpoint.go:378 msg="writing series" size="17 kB" level=info ts=2026-04-25T00:12:52.882183412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:12:52.886296363Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:12:54.617143655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:12:54.620617207Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" ts=2026-04-25T00:12:54.89054343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:12:55.40780788Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB" level=info ts=2026-04-25T00:12:55.449873511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:12:55.453352682Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-25T00:12:57.073253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:58.03045759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-25T00:12:58.741725143Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB" level=debug ts=2026-04-25T00:12:58.741748587Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=8 level=info ts=2026-04-25T00:13:00.186118775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:13:00.190284293Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:13:02.072804757Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:13:02.072829288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:13:02.072840315Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:13:02.072850529Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-25T00:13:02.081875615Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000018.tmp new=/var/loki/wal/checkpoint.000018 level=info ts=2026-04-25T00:13:02.083080617Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009471742s level=info ts=2026-04-25T00:13:03.623160746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:13:03.628181344Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-25T00:13:05.58796575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:13:05.593309552Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=warn ts=2026-04-25T00:13:07.072673548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:08.590545997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:13:08.595860054Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:13:08.666094851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:13:08.670271326Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:13:09.162735029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:13:09.166078895Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:13:11.57374235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:13:11.580671213Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:13:12.072673361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:13:12.66719764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=warn ts=2026-04-25T00:13:17.072480944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:18.516518639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:13:18.520745358Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:13:19.844738986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:13:19.855013931Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-25T00:13:22.073461579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:13:24.387271735Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:13:24.58026289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:13:24.58397098Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" ts=2026-04-25T00:13:24.892735229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=info ts=2026-04-25T00:13:26.763252429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:13:26.770454428Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:13:27.072645197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:27.348706398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:13:27.356068149Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:13:28.044446705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:13:28.050600644Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:13:28.117952052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:13:28.121773485Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:13:29.775607943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:13:29.779153784Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:13:30.85013605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:13:30.854010051Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:13:31.868943491Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:13:31.869905363Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:13:32.073126494Z caller=checkpoint.go:612 msg="starting checkpoint" level=info ts=2026-04-25T00:13:32.073148515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:13:32.073172346Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:13:32.073184091Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:13:32.073198788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:32.073492361Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000019 level=debug ts=2026-04-25T00:13:32.075912734Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB" level=debug ts=2026-04-25T00:13:32.075925Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=1 ts=2026-04-25T00:13:32.307225737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:54306" level=info ts=2026-04-25T00:13:32.390823845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:13:32.39527493Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:13:33.313275829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:13:33.31698419Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:13:34.192114277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:13:34.197682592Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-25T00:13:35.407924071Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB" level=info ts=2026-04-25T00:13:36.240757343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:13:36.246100236Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-25T00:13:37.072535458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:13:38.741270214Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB" level=warn ts=2026-04-25T00:13:42.072666407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:13:42.073780905Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" ts=2026-04-25T00:13:42.14584988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:44780" level=debug ts=2026-04-25T00:13:45.407931558Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=warn ts=2026-04-25T00:13:47.072754542Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:13:48.741443533Z caller=checkpoint.go:378 msg="writing series" size="151 kB" level=info ts=2026-04-25T00:13:49.139558617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:13:49.147249149Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=debug ts=2026-04-25T00:13:49.723054971Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:13:52.073072128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:13:52.074192012Z caller=checkpoint.go:378 msg="writing series" size="18 kB" level=info ts=2026-04-25T00:13:52.278110613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:13:52.28170868Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" ts=2026-04-25T00:13:54.894023228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:13:55.407501391Z caller=checkpoint.go:378 msg="writing series" size="25 kB" level=info ts=2026-04-25T00:13:55.462616346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:13:55.466767083Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:13:55.507829553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:13:55.511344776Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:13:56.641605242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:13:56.651939531Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:13:57.072947872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:57.837869915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:13:57.842319432Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:13:58.165498631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:13:58.169441322Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=debug ts=2026-04-25T00:13:58.741110069Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB" level=info ts=2026-04-25T00:14:02.072364899Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:14:02.072402186Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:14:02.072413079Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:14:02.072431679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:02.074620059Z caller=checkpoint.go:378 msg="writing series" size="202 kB" level=info ts=2026-04-25T00:14:02.3802886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:14:02.38506321Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-25T00:14:02.546004258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:14:02.549693577Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:14:05.205766294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:14:05.210919435Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=debug ts=2026-04-25T00:14:05.409240379Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB" level=debug ts=2026-04-25T00:14:05.409264399Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=10 level=warn ts=2026-04-25T00:14:07.073384308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:08.740975206Z caller=checkpoint.go:378 msg="writing series" size="194 kB" level=info ts=2026-04-25T00:14:08.976581804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:14:08.982285515Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:14:11.30681917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:14:11.310788475Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-25T00:14:12.073001834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:12.074558103Z caller=checkpoint.go:378 msg="writing series" size="197 kB" ts=2026-04-25T00:14:12.147733039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:44338" level=debug ts=2026-04-25T00:14:13.181216982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-25T00:14:14.378884549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:14:14.382984338Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:14:15.0556201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:14:15.063161958Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:14:15.275972784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:14:15.27956955Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-25T00:14:15.407473506Z caller=checkpoint.go:378 msg="writing series" size="48 kB" level=info ts=2026-04-25T00:14:16.650808924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:14:16.657676822Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:14:17.07328992Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:18.740669226Z caller=checkpoint.go:378 msg="writing series" size="496 kB" level=info ts=2026-04-25T00:14:19.171975277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:14:19.181162897Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-25T00:14:19.723963217Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:14:22.072932648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:22.082359344Z caller=checkpoint.go:378 msg="writing series" size="14 MB" level=debug ts=2026-04-25T00:14:22.082381917Z caller=checkpoint.go:390 msg="flushing series" totalSize="15 MB" series=5 level=info ts=2026-04-25T00:14:22.867474831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:14:22.872271349Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:14:23.555104909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:14:23.559177303Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:14:23.965180398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:14:23.969259701Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" ts=2026-04-25T00:14:24.896057591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:14:25.407419451Z caller=checkpoint.go:378 msg="writing series" size="63 kB" level=warn ts=2026-04-25T00:14:27.073382444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:28.740891889Z caller=checkpoint.go:378 msg="writing series" size="97 kB" level=info ts=2026-04-25T00:14:28.74493579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:14:28.748936738Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:14:30.539445019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:14:30.542871108Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:14:30.586068541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:14:30.589560499Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-25T00:14:31.868511416Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:14:31.869556271Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-25T00:14:32.07284111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:14:32.072877513Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:14:32.072894791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:14:32.072954446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:32.074024235Z caller=checkpoint.go:378 msg="writing series" size="51 kB" level=debug ts=2026-04-25T00:14:35.407510954Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB" level=warn ts=2026-04-25T00:14:37.072861558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:38.74097795Z caller=checkpoint.go:378 msg="writing series" size="5.1 kB" level=info ts=2026-04-25T00:14:38.927729294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:14:38.937411997Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:14:40.52591702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:14:40.530128459Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=warn ts=2026-04-25T00:14:42.072837908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:42.073875059Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=info ts=2026-04-25T00:14:43.39174233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:14:43.395818422Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-25T00:14:45.40725672Z caller=checkpoint.go:378 msg="writing series" size="122 kB" level=debug ts=2026-04-25T00:14:45.419389868Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:14:46.840376641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:14:46.84477369Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=warn ts=2026-04-25T00:14:47.07352344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:47.795377843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:14:47.79964536Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:14:48.741398911Z caller=checkpoint.go:378 msg="writing series" size="261 kB" level=info ts=2026-04-25T00:14:49.437919676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:14:49.444686965Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:14:51.708367938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:14:51.711564927Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-25T00:14:52.072836906Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:52.074028343Z caller=checkpoint.go:378 msg="writing series" size="191 kB" level=debug ts=2026-04-25T00:14:54.022690048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-25T00:14:54.024466247Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-25T00:14:54.89872746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:14:55.40794246Z caller=checkpoint.go:378 msg="writing series" size="112 kB" level=warn ts=2026-04-25T00:14:57.072877134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:58.408826163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:14:58.415005617Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-25T00:14:58.500386987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:14:58.504780425Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-25T00:14:58.741908827Z caller=checkpoint.go:378 msg="writing series" size="560 kB" level=debug ts=2026-04-25T00:14:58.741938442Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=11 level=info ts=2026-04-25T00:15:00.187439098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:15:00.192402455Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:15:00.429029302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:15:00.433490586Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-25T00:15:02.072806151Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:15:02.072806909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:02.072839655Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:15:02.072850087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:15:02.074255559Z caller=checkpoint.go:378 msg="writing series" size="529 kB" level=info ts=2026-04-25T00:15:02.15809721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:15:02.163507102Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:15:04.255296318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:15:04.260584938Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:15:04.404991527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:15:04.408908361Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=debug ts=2026-04-25T00:15:05.408168939Z caller=checkpoint.go:378 msg="writing series" size="415 kB" level=info ts=2026-04-25T00:15:06.599656111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:15:06.606846047Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:15:07.073159693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:08.740477419Z caller=checkpoint.go:378 msg="writing series" size="195 kB" level=debug ts=2026-04-25T00:15:08.740501512Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3 level=debug ts=2026-04-25T00:15:11.610999747Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:15:12.07321453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:12.074429242Z caller=checkpoint.go:378 msg="writing series" size="41 kB" level=info ts=2026-04-25T00:15:13.869143621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:15:13.87842889Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:15:14.53590789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:15:14.543804783Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:15:14.806840265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:15:14.81053242Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-25T00:15:15.408076924Z caller=checkpoint.go:378 msg="writing series" size="109 kB" level=info ts=2026-04-25T00:15:15.774137928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:15:15.778182467Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:15:16.331866337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:15:16.335463507Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:15:17.073178387Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:15:17.904390363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:15:17.909224295Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:15:18.206201774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:15:18.210464492Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:15:18.741603382Z caller=checkpoint.go:378 msg="writing series" size="172 kB" level=warn ts=2026-04-25T00:15:22.072544001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:22.074712618Z caller=checkpoint.go:378 msg="writing series" size="221 kB" ts=2026-04-25T00:15:24.901492615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=debug ts=2026-04-25T00:15:25.407250358Z caller=checkpoint.go:378 msg="writing series" size="125 kB" level=info ts=2026-04-25T00:15:26.01102438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:15:26.01552135Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:15:27.072516206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:28.741199174Z caller=checkpoint.go:378 msg="writing series" size="173 kB" level=info ts=2026-04-25T00:15:31.868686364Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:15:31.868867505Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=debug ts=2026-04-25T00:15:31.870876359Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads" level=debug ts=2026-04-25T00:15:31.870923496Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974528 n=1 level=debug ts=2026-04-25T00:15:31.870966672Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974528 n=1 level=info ts=2026-04-25T00:15:32.072750587Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:15:32.072788303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:15:32.072797878Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:15:32.072862626Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:32.073938678Z caller=checkpoint.go:378 msg="writing series" size="63 kB" level=info ts=2026-04-25T00:15:32.567371346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:15:32.571227917Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:15:33.795181587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:15:33.802653234Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:15:35.408000096Z caller=checkpoint.go:378 msg="writing series" size="19 kB" level=info ts=2026-04-25T00:15:36.023637064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:15:36.027616016Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:15:36.027643907Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:15:36.027677617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:15:36.031562249Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=warn ts=2026-04-25T00:15:37.07325149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:38.743401726Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB" level=debug ts=2026-04-25T00:15:38.743425477Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.4 MB" series=9 level=debug ts=2026-04-25T00:15:39.331168969Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:15:40.358111916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:15:40.362046533Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:15:41.241160114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-25T00:15:41.328560211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:15:41.3323032Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:15:41.491315942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:15:41.495614543Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=warn ts=2026-04-25T00:15:42.07334062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:42.07553405Z caller=checkpoint.go:378 msg="writing series" size="3.3 MB" level=debug ts=2026-04-25T00:15:42.075552523Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.3 MB" series=1 level=info ts=2026-04-25T00:15:42.208704922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:15:42.215227075Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=debug ts=2026-04-25T00:15:45.408757336Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB" level=info ts=2026-04-25T00:15:46.242144197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:15:46.24560058Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:15:46.610615618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:15:46.614382421Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:15:46.69248589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:15:46.695968715Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:15:46.730980821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:15:46.735564008Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=warn ts=2026-04-25T00:15:47.073050349Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:48.740961872Z caller=checkpoint.go:378 msg="writing series" size="23 kB" level=debug ts=2026-04-25T00:15:48.741002524Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=info ts=2026-04-25T00:15:49.071174759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:15:49.074699903Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:15:51.292959127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:15:51.297831612Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=warn ts=2026-04-25T00:15:52.073189897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:52.07426445Z caller=checkpoint.go:378 msg="writing series" size="939 B" ts=2026-04-25T00:15:53.510529627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:48200" level=info ts=2026-04-25T00:15:53.740623463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:15:53.746413588Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" ts=2026-04-25T00:15:54.903365934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:15:55.407063057Z caller=checkpoint.go:378 msg="writing series" size="817 B" level=info ts=2026-04-25T00:15:56.824852089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:15:56.833624388Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=warn ts=2026-04-25T00:15:57.072654707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:15:57.872658913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:15:57.88049164Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=debug ts=2026-04-25T00:15:58.741404732Z caller=checkpoint.go:378 msg="writing series" size="244 kB" level=info ts=2026-04-25T00:16:00.686538526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:16:00.690423803Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:16:02.073283411Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:16:02.073318216Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:16:02.073349563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:16:02.073360234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:02.075080633Z caller=checkpoint.go:378 msg="writing series" size="659 kB" level=info ts=2026-04-25T00:16:03.849449829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:16:03.854884559Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:16:03.892303208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:16:03.895666794Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-25T00:16:05.408590945Z caller=checkpoint.go:378 msg="writing series" size="609 kB" level=debug ts=2026-04-25T00:16:05.408616365Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5 level=warn ts=2026-04-25T00:16:07.07348314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:16:07.342715361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:16:07.350982275Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:16:07.853193486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:16:07.856722339Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-25T00:16:08.740776092Z caller=checkpoint.go:378 msg="writing series" size="149 kB" level=info ts=2026-04-25T00:16:08.774019792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:16:08.777485983Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:16:09.7402013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:16:09.745566464Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-25T00:16:12.073165641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:12.074558628Z caller=checkpoint.go:378 msg="writing series" size="382 kB" level=debug ts=2026-04-25T00:16:12.170715256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-25T00:16:15.407781585Z caller=checkpoint.go:378 msg="writing series" size="528 B" level=warn ts=2026-04-25T00:16:17.073102319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:18.741462165Z caller=checkpoint.go:378 msg="writing series" size="130 kB" level=info ts=2026-04-25T00:16:20.823182327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:16:20.833896292Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:16:20.833924007Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:16:20.833961009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:16:20.841515524Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:16:22.072469224Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:22.075343554Z caller=checkpoint.go:378 msg="writing series" size="410 kB" level=debug ts=2026-04-25T00:16:22.075368118Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5 level=info ts=2026-04-25T00:16:23.48637935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:16:23.490541115Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" ts=2026-04-25T00:16:23.513451878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:36348" level=debug ts=2026-04-25T00:16:24.436750127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-25T00:16:24.906215569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:16:25.407424185Z caller=checkpoint.go:378 msg="writing series" size="36 kB" level=info ts=2026-04-25T00:16:26.848011958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:16:26.852381215Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:16:26.9404478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="9.9 MB" avg_uncomp="9.9 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:16:26.945202757Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dc1a81a71%3A19dc1de8cc0%3A7682f872?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-25T00:16:26.945221954Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:16:26.945243969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="407 kB" avg_comp="407 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:16:26.948420674Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a81a1b%3A19dc1f8a31f%3A813702dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=warn ts=2026-04-25T00:16:27.073213761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:16:27.659591817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:16:27.663592432Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-25T00:16:28.740627714Z caller=checkpoint.go:378 msg="writing series" size="93 kB" level=info ts=2026-04-25T00:16:30.722005415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:16:30.725866267Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:16:31.868587722Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:16:31.869605498Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-25T00:16:32.07291453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:16:32.072916037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:32.072957157Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:16:32.072968912Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:16:32.074211149Z caller=checkpoint.go:378 msg="writing series" size="162 kB" level=debug ts=2026-04-25T00:16:35.407920964Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB" level=debug ts=2026-04-25T00:16:35.407945317Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4 level=info ts=2026-04-25T00:16:36.796883513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:16:36.802012692Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=warn ts=2026-04-25T00:16:37.072929633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:16:37.088079032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:16:37.096386215Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:16:37.490209406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:16:37.493651537Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=error ts=2026-04-25T00:16:37.493671078Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:16:37.493691933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:16:37.496987169Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:16:37.735782998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:16:37.739627914Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:16:37.739658594Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:16:37.793815345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:16:37.798063205Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-25T00:16:38.740494876Z caller=checkpoint.go:378 msg="writing series" size="119 kB" level=info ts=2026-04-25T00:16:39.854674703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:16:39.858658618Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:16:41.05158924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="408 kB" avg_comp="408 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:16:41.055613616Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a81a1b%3A19dc1f8a31f%3A813702dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=warn ts=2026-04-25T00:16:42.073012223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:42.131430566Z caller=checkpoint.go:378 msg="writing series" size="49 MB" level=debug ts=2026-04-25T00:16:42.131460859Z caller=checkpoint.go:390 msg="flushing series" totalSize="50 MB" series=2 level=info ts=2026-04-25T00:16:42.98563771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:16:42.990466283Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=debug ts=2026-04-25T00:16:44.399054244Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:16:44.552666643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:16:44.560778444Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:16:44.560804251Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:16:44.560836353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:16:44.567951864Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:16:45.334052243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:16:45.337883721Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=debug ts=2026-04-25T00:16:45.407303834Z caller=checkpoint.go:378 msg="writing series" size="676 kB" level=info ts=2026-04-25T00:16:45.573738202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:16:45.577847321Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:16:46.895442073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:16:46.901765891Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:16:46.950823889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:16:46.954229348Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:16:46.954248775Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:16:46.95427153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:16:46.95720966Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-25T00:16:47.072763247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:48.741779714Z caller=checkpoint.go:378 msg="writing series" size="468 kB" level=debug ts=2026-04-25T00:16:48.741828392Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=info ts=2026-04-25T00:16:50.708929254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:16:50.714233857Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:16:50.819532176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:16:50.826662633Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:16:50.843715532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:16:50.854405234Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:16:50.854440872Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:16:50.854495936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:16:50.864071482Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-25T00:16:51.821898886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:16:51.82557505Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-25T00:16:52.073523684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:52.074530745Z caller=checkpoint.go:378 msg="writing series" size="20 kB" level=info ts=2026-04-25T00:16:53.847602398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:16:53.851809555Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" ts=2026-04-25T00:16:54.909452323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:16:55.408272453Z caller=checkpoint.go:378 msg="writing series" size="606 kB" level=info ts=2026-04-25T00:16:55.813161473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:16:55.816923716Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:16:57.072921306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:16:57.127060196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:16:57.132127246Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:16:57.132154095Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:16:57.132189727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:16:57.135305021Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:16:57.529492648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:16:57.538588712Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-25T00:16:58.741223023Z caller=checkpoint.go:378 msg="writing series" size="13 kB" level=info ts=2026-04-25T00:17:02.072656738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:17:02.072671982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:02.072687109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:17:02.072696363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:17:02.073808007Z caller=checkpoint.go:378 msg="writing series" size="6.6 kB" level=debug ts=2026-04-25T00:17:03.497029299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-25T00:17:04.317190316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:17:04.321057966Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=info ts=2026-04-25T00:17:04.839165701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:17:04.842775251Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-25T00:17:05.407930486Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=info ts=2026-04-25T00:17:06.736768908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:17:06.74474003Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:17:07.073375948Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:17:08.503886444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:17:08.508037802Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:17:08.508064114Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:08.508096668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:17:08.511604147Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:17:08.741438713Z caller=checkpoint.go:378 msg="writing series" size="199 kB" level=info ts=2026-04-25T00:17:10.534667238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:17:10.542999476Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=warn ts=2026-04-25T00:17:12.073168614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:12.07430622Z caller=checkpoint.go:378 msg="writing series" size="158 kB" level=debug ts=2026-04-25T00:17:12.673615504Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:17:14.7354721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:17:14.739427205Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-25T00:17:15.407106342Z caller=checkpoint.go:378 msg="writing series" size="68 kB" level=debug ts=2026-04-25T00:17:15.407134617Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8 level=info ts=2026-04-25T00:17:15.588799421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="407 kB" avg_comp="407 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:17:15.593611463Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a81a1b%3A19dc1f8a31f%3A813702dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=info ts=2026-04-25T00:17:16.571462857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:17:16.574881965Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:17:17.072530203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:18.741132153Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=info ts=2026-04-25T00:17:19.132472578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:17:19.136367896Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:17:19.842808414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:17:19.846493486Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:17:19.84651892Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:19.846550302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:17:19.849582552Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=warn ts=2026-04-25T00:17:22.073463796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:22.074689042Z caller=checkpoint.go:378 msg="writing series" size="191 kB" ts=2026-04-25T00:17:23.518281222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:50306" ts=2026-04-25T00:17:24.912320225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-25T00:17:25.066345918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:17:25.071682335Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:17:25.407906147Z caller=checkpoint.go:378 msg="writing series" size="488 kB" level=info ts=2026-04-25T00:17:25.544985041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:17:25.54896061Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:17:26.133224049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:17:26.137789488Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=warn ts=2026-04-25T00:17:27.072860214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:28.74061945Z caller=checkpoint.go:378 msg="writing series" size="416 kB" level=debug ts=2026-04-25T00:17:28.740643824Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4 level=info ts=2026-04-25T00:17:29.772045763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:17:29.776176056Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:17:29.776199705Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:29.776225483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:17:29.779440335Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:17:30.159587538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:17:30.16322172Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:17:31.178192416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:17:31.182161496Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:17:31.188204373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:17:31.193080488Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:17:31.1931015Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:31.193127771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:17:31.198523208Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:17:31.869053734Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:17:31.869165644Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:17:32.07309143Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:17:32.073124359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:17:32.073133521Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:17:32.073154444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:32.074813025Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB" level=debug ts=2026-04-25T00:17:32.074829994Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=1 level=info ts=2026-04-25T00:17:33.326878032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:17:33.330718966Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:17:33.330748044Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:33.330778987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:17:33.334152417Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:17:33.968301143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:17:33.971969625Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:17:33.989059315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:17:33.992506906Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:17:35.407894846Z caller=checkpoint.go:378 msg="writing series" size="816 kB" level=warn ts=2026-04-25T00:17:37.07335609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:17:37.092466743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:17:37.101562082Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-25T00:17:38.741013629Z caller=checkpoint.go:378 msg="writing series" size="61 kB" level=debug ts=2026-04-25T00:17:41.348364561Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:17:41.376541079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:17:41.380375211Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=warn ts=2026-04-25T00:17:42.072725229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:42.073836851Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" ts=2026-04-25T00:17:42.168994315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37194" level=info ts=2026-04-25T00:17:42.781378283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:17:42.785139938Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:17:42.846235998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:17:42.849693629Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:17:42.849714562Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:42.849736092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:17:42.852551961Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:17:43.398609538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:17:43.4019359Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:17:43.401954884Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:43.401975402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:17:43.405414137Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=debug ts=2026-04-25T00:17:45.40758502Z caller=checkpoint.go:378 msg="writing series" size="16 kB" level=info ts=2026-04-25T00:17:46.831831169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:17:46.841198163Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:17:46.841236867Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:46.841279256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:17:46.849712994Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:17:47.025561299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:17:47.030133823Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-25T00:17:47.072856344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:17:47.260152491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:17:47.263659667Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:17:48.740933599Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB" level=info ts=2026-04-25T00:17:49.355316099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:17:49.360766103Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:17:49.999394708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:17:50.006088325Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-25T00:17:50.172082663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:17:50.175352955Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-25T00:17:52.073114618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:52.074229375Z caller=checkpoint.go:378 msg="writing series" size="17 kB" level=debug ts=2026-04-25T00:17:54.380494094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-25T00:17:54.914084603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d48aaaeb 10.130.0.59:7946" level=debug ts=2026-04-25T00:17:55.407588557Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB" level=info ts=2026-04-25T00:17:56.427555774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:17:56.432751342Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:17:56.952116894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:17:56.960602128Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:17:57.073197874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:17:57.246568243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:17:57.250025492Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:17:57.250046192Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:57.250069227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:17:57.252866914Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:17:57.311380419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="408 kB" avg_comp="408 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:17:57.315352958Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a81a1b%3A19dc1f8a31f%3A813702dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=info ts=2026-04-25T00:17:58.442097046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:17:58.446141288Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=debug ts=2026-04-25T00:17:58.743048389Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB" level=debug ts=2026-04-25T00:17:58.743074474Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 MB" series=8 level=info ts=2026-04-25T00:18:00.768289026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:18:00.772232487Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:18:02.072756974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:18:02.072785663Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:18:02.072793925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:18:02.07280168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:02.080499917Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000019.tmp new=/var/loki/wal/checkpoint.000019 level=info ts=2026-04-25T00:18:02.081597253Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007963249s level=info ts=2026-04-25T00:18:02.432448142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:18:02.436074948Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:18:05.445783174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:18:05.453173453Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:18:07.073461347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:09.864279846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:18:09.869774362Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-25T00:18:10.31305774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:18:11.386367829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:18:11.390053703Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-25T00:18:12.073291908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:18:12.170388264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:38946" level=info ts=2026-04-25T00:18:13.83545783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:18:13.840140238Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:18:13.840165384Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:13.840191398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:18:13.84406193Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:18:13.905504491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:18:13.909796447Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-25T00:18:17.072741124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:17.49783328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:18:17.501648772Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:18:19.30775687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:18:19.311889533Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:18:19.311908733Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:19.311930283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:18:19.315436727Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-25T00:18:20.905314829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:18:20.908891588Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:18:20.966042991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:18:20.971041267Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:18:22.073251508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:18:23.522152273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:57296" level=info ts=2026-04-25T00:18:23.717937551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:18:23.722587426Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:18:24.727520584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:18:24.731102415Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" ts=2026-04-25T00:18:24.916764491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=info ts=2026-04-25T00:18:25.435309117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:18:25.439768368Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:18:26.539547415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:18:26.546695218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:18:27.07274252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:29.12595945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:18:29.130044128Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:18:29.130067204Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:29.130094388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:18:29.13331164Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:18:29.237538846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:18:29.24115177Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:18:29.707217471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:18:29.710605498Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:18:29.967520856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:18:29.971079157Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:18:30.703145966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:18:30.71024927Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:18:31.868656534Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:18:31.869381999Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:18:32.072535127Z caller=checkpoint.go:612 msg="starting checkpoint" level=warn ts=2026-04-25T00:18:32.072573012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:32.072593056Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:18:32.072611733Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:18:32.07262044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-25T00:18:32.07281618Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000020 level=debug ts=2026-04-25T00:18:32.07417402Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB" level=debug ts=2026-04-25T00:18:32.074183163Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.3 MB" series=1 ts=2026-04-25T00:18:32.326979579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41846" level=info ts=2026-04-25T00:18:32.968454491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:18:32.975741044Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-25T00:18:33.652481407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:18:33.656605529Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=debug ts=2026-04-25T00:18:35.407334531Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB" level=warn ts=2026-04-25T00:18:37.072912888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:37.649796839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-25T00:18:38.739999505Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB" level=info ts=2026-04-25T00:18:39.864189141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="408 kB" avg_comp="408 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:18:39.868310872Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a81a1b%3A19dc1f8a31f%3A813702dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=info ts=2026-04-25T00:18:41.242147441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:18:41.246902536Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:18:42.013490822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:18:42.016534549Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-25T00:18:42.07292982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:42.072980932Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" ts=2026-04-25T00:18:42.171304879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:54396" level=info ts=2026-04-25T00:18:45.219954619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:18:45.224460076Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=debug ts=2026-04-25T00:18:45.407283194Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=info ts=2026-04-25T00:18:47.019372309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:18:47.023400726Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=warn ts=2026-04-25T00:18:47.072832697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:48.739756443Z caller=checkpoint.go:378 msg="writing series" size="208 kB" level=warn ts=2026-04-25T00:18:52.07296062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:52.073019227Z caller=checkpoint.go:378 msg="writing series" size="20 kB" level=debug ts=2026-04-25T00:18:53.050067286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-25T00:18:53.051773345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-25T00:18:54.919158309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:18:55.406909606Z caller=checkpoint.go:378 msg="writing series" size="25 kB" level=info ts=2026-04-25T00:18:55.470172592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:18:55.476604643Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:18:57.073438322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:57.936137099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:18:57.940757982Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:18:58.739729934Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB" level=info ts=2026-04-25T00:19:02.073016276Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:19:02.073045768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:19:02.073054498Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:19:02.073082384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:02.073145814Z caller=checkpoint.go:378 msg="writing series" size="202 kB" level=info ts=2026-04-25T00:19:02.726381739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:19:02.730054257Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:19:04.587622522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:19:04.592523715Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=debug ts=2026-04-25T00:19:05.407937528Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB" level=debug ts=2026-04-25T00:19:05.407958846Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.0 MB" series=10 level=warn ts=2026-04-25T00:19:07.07336013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:19:08.675986987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:19:08.679248371Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-25T00:19:08.74049682Z caller=checkpoint.go:378 msg="writing series" size="198 kB" level=info ts=2026-04-25T00:19:08.845636038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:19:08.848907185Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:19:09.401298732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:19:09.406151198Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:19:11.5131372Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:19:12.072643802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:12.073820802Z caller=checkpoint.go:378 msg="writing series" size="296 kB" ts=2026-04-25T00:19:12.172852702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55126" level=info ts=2026-04-25T00:19:12.83362827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:19:12.83902692Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:19:14.656435208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:19:14.66303113Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-25T00:19:15.406867151Z caller=checkpoint.go:378 msg="writing series" size="49 kB" level=info ts=2026-04-25T00:19:15.656764941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:19:15.662171423Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:19:16.401381914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:19:16.404915815Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-25T00:19:17.073212609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:18.740669986Z caller=checkpoint.go:378 msg="writing series" size="496 kB" level=info ts=2026-04-25T00:19:19.937455546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:19:19.947440924Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:19:20.083990731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:19:20.087964421Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:19:20.537111944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:19:20.540771736Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:19:21.386445751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:19:21.390640384Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-25T00:19:22.072822664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:22.08409055Z caller=checkpoint.go:378 msg="writing series" size="15 MB" level=debug ts=2026-04-25T00:19:22.084116926Z caller=checkpoint.go:390 msg="flushing series" totalSize="16 MB" series=5 level=info ts=2026-04-25T00:19:22.21239881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:19:22.216407654Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=info ts=2026-04-25T00:19:23.673859379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:19:23.677248515Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" ts=2026-04-25T00:19:24.922803676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-25T00:19:24.925149824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:19:24.928507987Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:19:24.92852699Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:19:24.928550549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:19:24.931354152Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=debug ts=2026-04-25T00:19:25.40689441Z caller=checkpoint.go:378 msg="writing series" size="63 kB" level=info ts=2026-04-25T00:19:25.920368706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:19:25.924079934Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=info ts=2026-04-25T00:19:26.454469178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:19:26.458381711Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=warn ts=2026-04-25T00:19:27.0729351Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:28.740672663Z caller=checkpoint.go:378 msg="writing series" size="97 kB" level=info ts=2026-04-25T00:19:31.712745879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:19:31.719767143Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" level=info ts=2026-04-25T00:19:31.868721435Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:19:31.868775188Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:19:32.072881679Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:19:32.072911962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:19:32.07292174Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:19:32.072922048Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:32.073003832Z caller=checkpoint.go:378 msg="writing series" size="51 kB" ts=2026-04-25T00:19:32.333771795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:46160" level=debug ts=2026-04-25T00:19:35.40672322Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB" level=debug ts=2026-04-25T00:19:36.330693437Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:19:36.900706063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="408 kB" avg_comp="408 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:19:36.905038018Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a81a1b%3A19dc1f8a31f%3A813702dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=warn ts=2026-04-25T00:19:37.072578685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:19:37.899029508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:19:37.902720333Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-25T00:19:37.968955299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:19:37.976372445Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:19:38.740304663Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB" level=warn ts=2026-04-25T00:19:42.072673388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:42.07376003Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB" level=debug ts=2026-04-25T00:19:45.407441682Z caller=checkpoint.go:378 msg="writing series" size="122 kB" level=warn ts=2026-04-25T00:19:47.073220514Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:47.201380716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-25T00:19:47.972514624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:19:47.976587475Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=debug ts=2026-04-25T00:19:48.740701775Z caller=checkpoint.go:378 msg="writing series" size="261 kB" level=info ts=2026-04-25T00:19:49.954857096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:19:49.959056124Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=warn ts=2026-04-25T00:19:52.073345581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:52.073426067Z caller=checkpoint.go:378 msg="writing series" size="191 kB" level=info ts=2026-04-25T00:19:52.088421162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:19:52.091916849Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" ts=2026-04-25T00:19:53.527974329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:60990" ts=2026-04-25T00:19:54.925278649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=debug ts=2026-04-25T00:19:55.406765908Z caller=checkpoint.go:378 msg="writing series" size="115 kB" level=info ts=2026-04-25T00:19:56.997980178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:19:57.003214393Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:19:57.072566914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:58.740582373Z caller=checkpoint.go:378 msg="writing series" size="638 kB" level=debug ts=2026-04-25T00:19:58.740607366Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=11 level=info ts=2026-04-25T00:20:00.838559144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:20:00.842138934Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:20:01.427921178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:20:01.431882063Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:20:01.650388847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:20:01.654064011Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:20:02.072356348Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:20:02.072388022Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:20:02.072396592Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:20:02.072434527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:20:02.073766217Z caller=checkpoint.go:378 msg="writing series" size="529 kB" level=info ts=2026-04-25T00:20:03.090629899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:20:03.094266156Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:20:03.831153908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:20:03.835001727Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:20:04.341383196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:20:04.344817692Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:20:04.825012468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:20:04.829461204Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:20:05.406762828Z caller=checkpoint.go:378 msg="writing series" size="415 kB" level=info ts=2026-04-25T00:20:05.606292306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:20:05.609775461Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:20:06.016153492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:20:06.02280734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:20:07.072949376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:20:08.090387438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:20:08.09818578Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=info ts=2026-04-25T00:20:08.436006135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:20:08.43959374Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:20:08.740436845Z caller=checkpoint.go:378 msg="writing series" size="240 kB" level=debug ts=2026-04-25T00:20:08.740460737Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3 level=info ts=2026-04-25T00:20:11.141364141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:20:11.14511492Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=debug ts=2026-04-25T00:20:11.416184597Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:20:11.800357041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:20:11.804081396Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=debug ts=2026-04-25T00:20:12.073180279Z caller=checkpoint.go:378 msg="writing series" size="41 kB" level=warn ts=2026-04-25T00:20:12.073198822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:20:12.176980003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55426" level=debug ts=2026-04-25T00:20:15.407125453Z caller=checkpoint.go:378 msg="writing series" size="109 kB" level=info ts=2026-04-25T00:20:16.502138291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:20:16.507833192Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=warn ts=2026-04-25T00:20:17.072778152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:20:18.740610712Z caller=checkpoint.go:378 msg="writing series" size="180 kB" level=info ts=2026-04-25T00:20:19.390834414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:20:19.397426204Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:20:22.072447914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:20:22.073644563Z caller=checkpoint.go:378 msg="writing series" size="221 kB" level=info ts=2026-04-25T00:20:22.774372292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:20:22.778477899Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=info ts=2026-04-25T00:20:23.470581096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:20:23.477781944Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" ts=2026-04-25T00:20:23.530992712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:55566" ts=2026-04-25T00:20:24.928379431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946" level=debug ts=2026-04-25T00:20:25.406892877Z caller=checkpoint.go:378 msg="writing series" size="125 kB" level=warn ts=2026-04-25T00:20:27.073502645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:20:28.740529879Z caller=checkpoint.go:378 msg="writing series" size="198 kB" level=debug ts=2026-04-25T00:20:29.713422195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=info ts=2026-04-25T00:20:31.869370728Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:20:31.869841905Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:20:32.073422523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:20:32.073436512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:20:32.073449611Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:20:32.073458998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:20:32.073539147Z caller=checkpoint.go:378 msg="writing series" size="65 kB" level=info ts=2026-04-25T00:20:32.555432996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="408 kB" avg_comp="408 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:20:32.55920118Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a81a1b%3A19dc1f8a31f%3A813702dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=debug ts=2026-04-25T00:20:35.40653102Z caller=checkpoint.go:378 msg="writing series" size="19 kB" level=info ts=2026-04-25T00:20:36.784343105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:20:36.788118311Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=warn ts=2026-04-25T00:20:37.07303001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:20:38.742425485Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB" level=debug ts=2026-04-25T00:20:38.742446923Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.5 MB" series=9 level=debug ts=2026-04-25T00:20:41.085483401Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:20:41.085560079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="376 kB" avg_comp="376 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-25T00:20:41.089597553Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dc1a8208e%3A19dc20086b0%3A604d8cac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=warn ts=2026-04-25T00:20:42.073383772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:20:42.075106028Z caller=checkpoint.go:378 msg="writing series" size="3.4 MB" level=debug ts=2026-04-25T00:20:42.075122768Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.4 MB" series=1 ts=2026-04-25T00:20:42.178610233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:47508" level=info ts=2026-04-25T00:20:44.145156233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:20:44.149150191Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:20:44.238814417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:20:44.243059961Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=debug ts=2026-04-25T00:20:45.407652388Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB" level=info ts=2026-04-25T00:20:46.672056326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:20:46.67598593Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:20:47.072899178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:20:47.726372348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:20:47.729727408Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a8177b%3A19dc1a875a6%3A35eaa9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=debug ts=2026-04-25T00:20:48.740483394Z caller=checkpoint.go:378 msg="writing series" size="23 kB" level=debug ts=2026-04-25T00:20:48.740509049Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2 level=info ts=2026-04-25T00:20:48.845767911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:20:48.84938269Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a875a5%3A19dc1a8c5d0%3A61d425aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:20:49.213142974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:20:49.216601147Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:20:49.318203003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="114 MB" avg_uncomp="19 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:20:49.326623283Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a81bd6%3A19dc1a8b33f%3Ab01e84d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:20:49.696637158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:20:49.69998555Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a81205%3A19dc1a875a6%3Aca15ab77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-25T00:20:52.073003567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:20:52.073046235Z caller=checkpoint.go:378 msg="writing series" size="939 B" ts=2026-04-25T00:20:53.533896983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:58124" level=info ts=2026-04-25T00:20:53.934667198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:20:53.938304044Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a875a5%3A19dc1a875a6%3Ac845e5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:20:54.304270683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:20:54.309574915Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a82150%3A19dc1d207bd%3A72e1ab3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" ts=2026-04-25T00:20:54.931445865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=info ts=2026-04-25T00:20:55.131281095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:20:55.13522267Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b3ba87%3A19dc1b52b79%3A1d93b083?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-25T00:20:55.406699297Z caller=checkpoint.go:378 msg="writing series" size="817 B" level=info ts=2026-04-25T00:20:55.516121609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}" level=error ts=2026-04-25T00:20:55.51942466Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19dc1a8a6b8%3A19dc1a8a6b9%3A6d74d278?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}" level=warn ts=2026-04-25T00:20:57.073253222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:20:57.259632722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-25T00:20:57.263487831Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dc1a8208e%3A19dc20086b0%3A604d8cac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=info ts=2026-04-25T00:20:57.43763716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-25T00:20:57.440856807Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dc1a875a5%3A19dc1be3a09%3A3aba25d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=debug ts=2026-04-25T00:20:58.739973729Z caller=checkpoint.go:378 msg="writing series" size="262 kB" level=info ts=2026-04-25T00:21:01.353635952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="469 kB" avg_comp="469 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:21:01.357647127Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a819b0%3A19dc1c20a8f%3A6f2f7222?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:21:02.002951483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:21:02.006407802Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a875a5%3A19dc1a875a6%3A38ba6b88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:21:02.072752747Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:21:02.072776043Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:21:02.072786144Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:21:02.072805169Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:21:02.074120273Z caller=checkpoint.go:378 msg="writing series" size="670 kB" level=info ts=2026-04-25T00:21:04.634092846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=de55ccc6c90b5a27 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-qsjr\"}" level=error ts=2026-04-25T00:21:04.641896154Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de55ccc6c90b5a27/19dc1b3ed47%3A19dc1c0aa81%3Aa5210079?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-qsjr\"}" level=debug ts=2026-04-25T00:21:05.4076676Z caller=checkpoint.go:378 msg="writing series" size="609 kB" level=debug ts=2026-04-25T00:21:05.407690502Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5 level=info ts=2026-04-25T00:21:06.269782168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e87c5ed1b5e84ecc immediate=false num_chunks=1 total_comp="382 kB" avg_comp="382 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"gitlab-rep-oela\"}" level=error ts=2026-04-25T00:21:06.274895416Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e87c5ed1b5e84ecc/19dc1b3cfd0%3A19dc1c0b94f%3A38dad241?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-oela\"}" level=info ts=2026-04-25T00:21:06.784153003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8801bcba29d03809 immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="852 kB" avg_uncomp="852 kB" full=1 labels="{stream=\"stat-rep-wmty\"}" level=error ts=2026-04-25T00:21:06.787688651Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8801bcba29d03809/19dc1b3f2f8%3A19dc1bcf1df%3Ae997285a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-wmty\"}" level=warn ts=2026-04-25T00:21:07.072924026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:21:08.740557314Z caller=checkpoint.go:378 msg="writing series" size="200 kB" level=debug ts=2026-04-25T00:21:11.848641046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=warn ts=2026-04-25T00:21:12.072805011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:21:12.07401384Z caller=checkpoint.go:378 msg="writing series" size="382 kB" level=info ts=2026-04-25T00:21:12.817708003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.3 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:21:12.825133787Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a81248%3A19dc1c31139%3A373cb76e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:21:13.520986779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="408 kB" avg_comp="408 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:21:13.524877986Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a81a1b%3A19dc1f8a31f%3A813702dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=debug ts=2026-04-25T00:21:15.406904039Z caller=checkpoint.go:378 msg="writing series" size="528 B" level=debug ts=2026-04-25T00:21:16.812240886Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:21:17.073187126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:21:18.739817245Z caller=checkpoint.go:378 msg="writing series" size="130 kB" level=warn ts=2026-04-25T00:21:22.073202983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:21:22.073427054Z caller=checkpoint.go:378 msg="writing series" size="410 kB" level=debug ts=2026-04-25T00:21:22.07344255Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5 level=info ts=2026-04-25T00:21:23.095143009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="393 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}" level=error ts=2026-04-25T00:21:23.105015943Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dc1a80fb4%3A19dc1b3af4f%3A1131f53b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}" ts=2026-04-25T00:21:24.934108597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946" level=debug ts=2026-04-25T00:21:25.406562515Z caller=checkpoint.go:378 msg="writing series" size="36 kB" level=info ts=2026-04-25T00:21:25.932154228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="337 kB" avg_comp="337 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:21:25.936335947Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a80ea2%3A19dc1f83d98%3Adaa02bec?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=warn ts=2026-04-25T00:21:27.072923068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:21:28.740638756Z caller=checkpoint.go:378 msg="writing series" size="99 kB" level=info ts=2026-04-25T00:21:29.699138066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="619 kB" avg_comp="619 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:21:29.703508729Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a817c6%3A19dc1f1f048%3Af30e59f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-25T00:21:31.732020762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-25T00:21:31.737463211Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dc1a8208e%3A19dc20086b0%3A604d8cac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=info ts=2026-04-25T00:21:31.868753369Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:21:31.868773061Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1 level=info ts=2026-04-25T00:21:32.073024871Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:21:32.07305252Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:21:32.073063428Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:21:32.07306918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:21:32.073138614Z caller=checkpoint.go:378 msg="writing series" size="168 kB" level=debug ts=2026-04-25T00:21:35.408423965Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB" level=debug ts=2026-04-25T00:21:35.408444946Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=4 level=warn ts=2026-04-25T00:21:37.072616881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:21:37.580029604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:21:37.583773059Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a87599%3A19dc1a875a6%3Ad08d6e4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:21:38.739757159Z caller=checkpoint.go:378 msg="writing series" size="118 kB" level=info ts=2026-04-25T00:21:39.394271074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:21:39.397906981Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a875a5%3A19dc1a8a6b9%3Acd7bda00?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:21:41.809374615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:21:41.813160432Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a875a5%3A19dc1a8b9ec%3A535a1666?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"