level=info ts=2026-04-23T09:43:20.837518985Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)" level=info ts=2026-04-23T09:43:20.837548712Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml level=debug ts=2026-04-23T09:43:20.837565308Z caller=modules.go:1984 msg="initializing ingester query tags interceptors" level=info ts=2026-04-23T09:43:20.83851893Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095 level=debug ts=2026-04-23T09:43:20.838910037Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-23T09:43:20.838987011Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.89 inf=eth0 level=info ts=2026-04-23T09:43:20.839458155Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-05c29a36 ts=2026-04-23T09:43:20.839537283Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired" level=debug ts=2026-04-23T09:43:20.839549423Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.89 advertisePort=7946 level=debug ts=2026-04-23T09:43:20.839567842Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.89 advertisePort=7946 level=info ts=2026-04-23T09:43:20.839693935Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4 level=debug ts=2026-04-23T09:43:20.841635916Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1 level=info ts=2026-04-23T09:43:20.842759246Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode" level=info ts=2026-04-23T09:43:20.842920682Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-23T09:43:20.843801878Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0 level=debug ts=2026-04-23T09:43:20.843820351Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-23T09:43:20.843819131Z level=debug ts=2026-04-23T09:43:20.843833955Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads" level=info ts=2026-04-23T09:43:20.843847445Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0 level=info ts=2026-04-23T09:43:20.843860654Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1 level=debug ts=2026-04-23T09:43:20.843864955Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-04-23T09:43:20.843864408Z level=debug ts=2026-04-23T09:43:20.843869115Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads" level=debug ts=2026-04-23T09:43:20.845199522Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-23T09:43:20.845293948Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.89 inf=eth0 ts=2026-04-23T09:43:20.847977835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.73:7946" ts=2026-04-23T09:43:20.851550358Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused" ts=2026-04-23T09:43:20.852137177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.129.0.88:7946" ts=2026-04-23T09:43:20.853516442Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.87:7946: dial tcp 10.129.0.87:7946: connect: connection refused" ts=2026-04-23T09:43:20.854706854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.130.0.65:7946" ts=2026-04-23T09:43:20.858016163Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.66:7946: dial tcp 10.130.0.66:7946: connect: connection refused" level=info ts=2026-04-23T09:43:20.85802767Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=18.336135ms level=info ts=2026-04-23T09:43:20.85804493Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local level=info ts=2026-04-23T09:43:20.85843809Z caller=module_service.go:82 msg=starting module=runtime-config level=info ts=2026-04-23T09:43:20.858447318Z caller=module_service.go:82 msg=starting module=server level=debug ts=2026-04-23T09:43:20.858625946Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server level=info ts=2026-04-23T09:43:20.85875543Z caller=module_service.go:82 msg=starting module=memberlist-kv level=debug ts=2026-04-23T09:43:20.85884362Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics level=debug ts=2026-04-23T09:43:20.858517334Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring level=debug ts=2026-04-23T09:43:20.858881509Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv level=debug ts=2026-04-23T09:43:20.858886924Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config level=debug ts=2026-04-23T09:43:20.858900999Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config level=debug ts=2026-04-23T09:43:20.858905333Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server level=debug ts=2026-04-23T09:43:20.858908222Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server level=debug ts=2026-04-23T09:43:20.858912645Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv level=info ts=2026-04-23T09:43:20.858911711Z caller=module_service.go:82 msg=starting module=store level=info ts=2026-04-23T09:43:20.858915856Z caller=module_service.go:82 msg=starting module=ring level=debug ts=2026-04-23T09:43:20.859017907Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config level=debug ts=2026-04-23T09:43:20.85902274Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server level=debug ts=2026-04-23T09:43:20.859025844Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv level=info ts=2026-04-23T09:43:20.859028766Z caller=module_service.go:82 msg=starting module=analytics level=debug ts=2026-04-23T09:43:20.859048175Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv level=debug ts=2026-04-23T09:43:20.859051704Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring level=debug ts=2026-04-23T09:43:20.859054519Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config level=debug ts=2026-04-23T09:43:20.859057247Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server level=debug ts=2026-04-23T09:43:20.859059921Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store level=info ts=2026-04-23T09:43:20.859062442Z caller=module_service.go:82 msg=starting module=ingester level=info ts=2026-04-23T09:43:20.859072327Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint" ts=2026-04-23T09:43:20.861692544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.73:7946" ts=2026-04-23T09:43:20.86390225Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused" ts=2026-04-23T09:43:20.864032236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.129.0.88:7946" ts=2026-04-23T09:43:20.864765225Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.87:7946: dial tcp 10.129.0.87:7946: connect: connection refused" ts=2026-04-23T09:43:20.865764476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.130.0.65:7946" ts=2026-04-23T09:43:20.867807987Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.66:7946: dial tcp 10.130.0.66:7946: connect: connection refused" level=info ts=2026-04-23T09:43:20.867818694Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=9.768641ms level=info ts=2026-04-23T09:43:21.039128979Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=180.057334ms errors=false level=info ts=2026-04-23T09:43:21.039155593Z caller=ingester.go:587 component=ingester msg="recovering from WAL" ts=2026-04-23T09:43:21.168093638Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-4fcc0c35' from=10.129.0.89:7946" ts=2026-04-23T09:43:21.170155338Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-4fcc0c35' from=10.130.0.65:7946" level=info ts=2026-04-23T09:43:21.744952147Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=885.880398ms errors=false level=info ts=2026-04-23T09:43:21.744982829Z caller=ingester.go:551 component=ingester msg="closing recoverer" level=info ts=2026-04-23T09:43:21.745016706Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=885.94531ms level=info ts=2026-04-23T09:43:21.745040239Z caller=wal.go:158 msg=started component=wal level=info ts=2026-04-23T09:43:21.745088893Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty" level=info ts=2026-04-23T09:43:21.745090931Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=19.874747028s level=info ts=2026-04-23T09:43:21.745132441Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z level=debug ts=2026-04-23T09:43:21.745162209Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester level=info ts=2026-04-23T09:43:21.745198015Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat" level=info ts=2026-04-23T09:43:21.745209739Z caller=loki.go:599 msg="Loki started" startup_time=965.912716ms ts=2026-04-23T09:43:22.781901079Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-4fcc0c35' from=10.130.0.65:7946" ts=2026-04-23T09:43:24.782993254Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-4fcc0c35' from=10.129.0.89:7946" ts=2026-04-23T09:43:24.783082349Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-4fcc0c35' from=10.129.0.88:7946" ts=2026-04-23T09:43:24.783539198Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-4fcc0c35' from=10.128.0.73:7946" level=warn ts=2026-04-23T09:43:26.746044328Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-23T09:43:31.746226239Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:43:35.183915705Z caller=logging.go:137 msg="GET /ready (503) 106.177µs" level=warn ts=2026-04-23T09:43:36.746014964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-23T09:43:41.746109629Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:43:43.089086732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:59692" level=debug ts=2026-04-23T09:43:45.182249629Z caller=logging.go:137 msg="GET /ready (503) 70.794µs" level=warn ts=2026-04-23T09:43:46.746120533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:43:48.353322426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:42696" ts=2026-04-23T09:43:48.638680324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40194" level=info ts=2026-04-23T09:43:51.745810158Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-23T09:43:51.745831329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:43:51.745850442Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership" level=info ts=2026-04-23T09:43:51.745860079Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000 level=info ts=2026-04-23T09:43:51.745871788Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-23T09:43:56.746068041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:43:57.030737771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946" level=warn ts=2026-04-23T09:44:01.745806294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-23T09:44:06.746107005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:44:09.978493638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9f37ee7ca8074ee0 immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="12 kB" avg_uncomp="12 kB" idle=1 labels="{stream=\"build-e2e-tqxo\"}" level=info ts=2026-04-23T09:44:09.978500823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="903 kB" avg_comp="903 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}" level=info ts=2026-04-23T09:44:09.978492234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=info ts=2026-04-23T09:44:09.978515072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"tekton-results\"}" level=info ts=2026-04-23T09:44:09.978579642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=info ts=2026-04-23T09:44:09.978626665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-23T09:44:09.978642867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=63fa18e885493035 immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="42 MB" avg_uncomp="10 MB" full=3 synced=1 labels="{stream=\"group-jsux\"}" level=info ts=2026-04-23T09:44:09.978681273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=info ts=2026-04-23T09:44:09.978934842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="696 kB" avg_comp="696 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=info ts=2026-04-23T09:44:09.978999814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=info ts=2026-04-23T09:44:09.979004544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=info ts=2026-04-23T09:44:09.979033235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2936a11c98a836c immediate=false num_chunks=1 total_comp="173 kB" avg_comp="173 kB" total_uncomp="859 kB" avg_uncomp="859 kB" idle=1 labels="{stream=\"stat-rep-sjil\"}" level=info ts=2026-04-23T09:44:09.979049534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-23T09:44:09.979061178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="487 kB" avg_comp="487 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}" level=info ts=2026-04-23T09:44:09.979083732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a0d2eed16cfb5aad immediate=false num_chunks=1 total_comp="198 B" avg_comp="198 B" total_uncomp="615 B" avg_uncomp="615 B" idle=1 labels="{stream=\"tenant-dev-cwev\"}" level=info ts=2026-04-23T09:44:09.979086762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0a598b07aac1932a immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-mnwl\"}" level=info ts=2026-04-23T09:44:09.979147072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}" level=info ts=2026-04-23T09:44:09.979155664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=debug ts=2026-04-23T09:44:09.979360431Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-23T09:44:09.979376294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-23T09:44:09.979368749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-23T09:44:09.979382948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=52e28635a1e69389 immediate=false num_chunks=1 total_comp="759 B" avg_comp="759 B" total_uncomp="25 kB" avg_uncomp="25 kB" idle=1 labels="{stream=\"chains-e2e-bnlr\"}" level=info ts=2026-04-23T09:44:09.979396536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="420 kB" avg_comp="420 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=info ts=2026-04-23T09:44:09.979395222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="36 kB" avg_uncomp="36 kB" idle=1 labels="{stream=\"happy-path-managed\"}" level=info ts=2026-04-23T09:44:09.97940623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}" level=info ts=2026-04-23T09:44:09.979403307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2da642124d384da8 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" idle=1 labels="{stream=\"konflux-ypno\"}" level=error ts=2026-04-23T09:44:09.984396117Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db962e8e4%3A19db993161d%3A918ee0bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=error ts=2026-04-23T09:44:09.984414112Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/52e28635a1e69389/19db96f329c%3A19db9761604%3Accb48298?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-bnlr\"}" level=error ts=2026-04-23T09:44:09.984412839Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a0d2eed16cfb5aad/19db96ed326%3A19db96ef035%3A33aed80e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-cwev\"}" level=error ts=2026-04-23T09:44:09.984414435Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db96333a3%3A19db9760eb7%3A84cd9a59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=error ts=2026-04-23T09:44:09.984418126Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9f37ee7ca8074ee0/19db96f888c%3A19db9714bf8%3A484e613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-tqxo\"}" level=error ts=2026-04-23T09:44:09.984420909Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db96333a3%3A19db9635646%3A3d32df0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=error ts=2026-04-23T09:44:09.984420906Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db96333a3%3A19db9635646%3A5ab093ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=error ts=2026-04-23T09:44:09.984456488Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db962eb43%3A19db96333a5%3Adc6ca458?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-23T09:44:09.984476583Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db9634c89%3A19db9634c8a%3A62f07f65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=error ts=2026-04-23T09:44:09.984481069Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db9634c89%3A19db9635646%3A62a82fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=error ts=2026-04-23T09:44:09.984481437Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db962e8f5%3A19db9afb404%3Ab41c8433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}" level=error ts=2026-04-23T09:44:09.984481695Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db962eb8a%3A19db9a07247%3A2ebb1595?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=error ts=2026-04-23T09:44:09.98448654Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19db962fc91%3A19db9a3c322%3Af163b89c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}" level=error ts=2026-04-23T09:44:09.984487139Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db962e775%3A19db9802088%3A905386ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-23T09:44:09.984479988Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db962e782%3A19db9635646%3Aecb163a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-23T09:44:09.984495392Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db96333a4%3A19db96399de%3A777b031b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-23T09:44:09.98446244Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db962e7a2%3A19db99e5cd6%3Ab5d3b01b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=error ts=2026-04-23T09:44:09.984509029Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db96f632b%3A19db970e04e%3A45b631d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=error ts=2026-04-23T09:44:09.984520748Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db962eaf7%3A19db9946c27%3Aec3e3ab9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-23T09:44:09.984529757Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a2936a11c98a836c/19db96f540d%3A19db9769b57%3Af0d6f9f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-sjil\"}" level=error ts=2026-04-23T09:44:09.986264841Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/63fa18e885493035/19db96f3c9e%3A19db97f85d3%3Ac8c91f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsux\"}" level=error ts=2026-04-23T09:44:09.986308608Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19db96333a3%3A19db9a6bba7%3Afdefc619?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=error ts=2026-04-23T09:44:09.986311163Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a598b07aac1932a/19db96f7504%3A19db9830708%3A4cfb3855?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-mnwl\"}" level=error ts=2026-04-23T09:44:09.986331596Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db96333a4%3A19db9a9e43d%3A88a81321?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-23T09:44:09.98891176Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2da642124d384da8/19db96fff6b%3A19db97e209d%3A8d10409c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ypno\"}" level=info ts=2026-04-23T09:44:10.279871746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="155 kB" avg_uncomp="155 kB" idle=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-23T09:44:10.283493809Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db962ea76%3A19db97c8e44%3A20dbb383?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=warn ts=2026-04-23T09:44:11.745733585Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:44:14.398396712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-04-23T09:44:16.745462293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:44:18.356521953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59192" ts=2026-04-23T09:44:18.641209167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40544" level=debug ts=2026-04-23T09:44:20.842811684Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1 level=info ts=2026-04-23T09:44:20.843838645Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-23T09:44:21.088876508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-23T09:44:21.092762515Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db96333a3%3A19db9635646%3A5ab093ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-23T09:44:21.277368645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0a598b07aac1932a immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-mnwl\"}" level=error ts=2026-04-23T09:44:21.282116086Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a598b07aac1932a/19db96f7504%3A19db9830708%3A4cfb3855?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-mnwl\"}" level=info ts=2026-04-23T09:44:21.566543583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=63fa18e885493035 immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="42 MB" avg_uncomp="10 MB" full=3 synced=1 labels="{stream=\"group-jsux\"}" level=error ts=2026-04-23T09:44:21.574179942Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/63fa18e885493035/19db96f3c9e%3A19db97f85d3%3Ac8c91f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsux\"}" level=info ts=2026-04-23T09:44:21.609252601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a0d2eed16cfb5aad immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-cwev\"}" level=error ts=2026-04-23T09:44:21.612344263Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a0d2eed16cfb5aad/19db96ed326%3A19db96ef035%3A33aed80e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-cwev\"}" level=info ts=2026-04-23T09:44:21.739932959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-23T09:44:21.743133079Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db962eb43%3A19db96333a5%3Adc6ca458?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-23T09:44:21.746187007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-23T09:44:21.746200778Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=warn ts=2026-04-23T09:44:21.746196936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:44:21.746210492Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-23T09:44:22.351463003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2da642124d384da8 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"konflux-ypno\"}" level=error ts=2026-04-23T09:44:22.355727317Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2da642124d384da8/19db96fff6b%3A19db97e209d%3A8d10409c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ypno\"}" level=info ts=2026-04-23T09:44:22.455129799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="484 kB" avg_comp="484 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-23T09:44:22.459119323Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db962e7a2%3A19db99e5cd6%3Ab5d3b01b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-23T09:44:22.77979946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="420 kB" avg_comp="420 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-23T09:44:22.783807762Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db962eb8a%3A19db9a07247%3A2ebb1595?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=info ts=2026-04-23T09:44:22.895298055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-23T09:44:22.898734223Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db96f632b%3A19db970e04e%3A45b631d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-23T09:44:23.168953334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-23T09:44:23.172396368Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db962ea76%3A19db97c8e44%3A20dbb383?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-23T09:44:23.400960534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9f37ee7ca8074ee0 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-tqxo\"}" level=error ts=2026-04-23T09:44:23.404643866Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9f37ee7ca8074ee0/19db96f888c%3A19db9714bf8%3A484e613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-tqxo\"}" level=info ts=2026-04-23T09:44:24.450142768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-23T09:44:24.455742398Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db962e775%3A19db9802088%3A905386ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-23T09:44:24.525233661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-23T09:44:24.5288979Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db9634c89%3A19db9634c8a%3A62f07f65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-23T09:44:24.819447683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="487 kB" avg_comp="487 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}" level=error ts=2026-04-23T09:44:24.823688609Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db962e8f5%3A19db9afb404%3Ab41c8433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}" level=info ts=2026-04-23T09:44:25.418656276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2936a11c98a836c immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="859 kB" avg_uncomp="859 kB" full=1 labels="{stream=\"stat-rep-sjil\"}" level=error ts=2026-04-23T09:44:25.422364055Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a2936a11c98a836c/19db96f540d%3A19db9769b57%3Af0d6f9f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-sjil\"}" level=info ts=2026-04-23T09:44:25.920284159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="903 kB" avg_comp="903 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}" level=error ts=2026-04-23T09:44:25.924962305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19db962fc91%3A19db9a3c322%3Af163b89c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}" level=info ts=2026-04-23T09:44:25.955193814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-23T09:44:25.958568962Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db96333a4%3A19db96399de%3A777b031b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-23T09:44:26.246452762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-23T09:44:26.253154815Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db96333a4%3A19db9a9e43d%3A88a81321?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-23T09:44:26.588102679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="696 kB" avg_comp="696 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-23T09:44:26.592439721Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db962e8e4%3A19db993161d%3A918ee0bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=warn ts=2026-04-23T09:44:26.745842267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:44:27.033181704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-7e16cc4f 10.129.0.88:7946" level=info ts=2026-04-23T09:44:27.539652649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=52e28635a1e69389 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-bnlr\"}" level=error ts=2026-04-23T09:44:27.54340332Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/52e28635a1e69389/19db96f329c%3A19db9761604%3Accb48298?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-bnlr\"}" level=info ts=2026-04-23T09:44:27.877484077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-23T09:44:27.881033565Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db96333a3%3A19db9760eb7%3A84cd9a59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-23T09:44:27.969291009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-23T09:44:27.973526664Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db962eaf7%3A19db9946c27%3Aec3e3ab9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-23T09:44:28.701645619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-23T09:44:28.708248257Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19db96333a3%3A19db9a6bba7%3Afdefc619?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-23T09:44:29.513167458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-23T09:44:29.517019168Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db96333a3%3A19db9635646%3A3d32df0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-23T09:44:29.729453607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-23T09:44:29.733380785Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db962e782%3A19db9635646%3Aecb163a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-23T09:44:29.845871104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-23T09:44:29.8494899Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db9634c89%3A19db9635646%3A62a82fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-23T09:44:31.391009119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-04-23T09:44:31.746096581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-23T09:44:36.746219732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-23T09:44:41.74562154Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:44:44.498113293Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-23T09:44:44.498174367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e010f61302c79b78 immediate=false num_chunks=1 total_comp="781 kB" avg_comp="781 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" idle=1 labels="{stream=\"build-e2e-zkra\"}" level=error ts=2026-04-23T09:44:44.511490368Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e010f61302c79b78/19db96ff79b%3A19db9833f6d%3A3ddabfc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zkra\"}" level=warn ts=2026-04-23T09:44:46.745464865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:44:47.105038249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-23T09:44:47.108372208Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db962eb43%3A19db96333a5%3Adc6ca458?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" ts=2026-04-23T09:44:48.359700463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56302" level=info ts=2026-04-23T09:44:49.043317193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="903 kB" avg_comp="903 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}" level=error ts=2026-04-23T09:44:49.04795635Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19db962fc91%3A19db9a3c322%3Af163b89c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}" level=info ts=2026-04-23T09:44:49.544399809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0a598b07aac1932a immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-mnwl\"}" level=error ts=2026-04-23T09:44:49.549497827Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a598b07aac1932a/19db96f7504%3A19db9830708%3A4cfb3855?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-mnwl\"}" level=info ts=2026-04-23T09:44:49.69938674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=63fa18e885493035 immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="42 MB" avg_uncomp="10 MB" full=3 synced=1 labels="{stream=\"group-jsux\"}" level=error ts=2026-04-23T09:44:49.706344706Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/63fa18e885493035/19db96f3c9e%3A19db97f85d3%3Ac8c91f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsux\"}" level=info ts=2026-04-23T09:44:49.891103163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-23T09:44:49.897946591Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db96333a4%3A19db9a9e43d%3A88a81321?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-23T09:44:50.426807547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-23T09:44:50.433136235Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19db96333a3%3A19db9a6bba7%3Afdefc619?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-23T09:44:51.745978044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-23T09:44:51.746009526Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-23T09:44:51.746017512Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-23T09:44:51.746030915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:44:51.883073465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=52e28635a1e69389 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-bnlr\"}" level=error ts=2026-04-23T09:44:51.886670698Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/52e28635a1e69389/19db96f329c%3A19db9761604%3Accb48298?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-bnlr\"}" level=info ts=2026-04-23T09:44:52.128591849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-23T09:44:52.132037553Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db962ea76%3A19db97c8e44%3A20dbb383?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-23T09:44:52.951659787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-23T09:44:52.955011888Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db96333a4%3A19db96399de%3A777b031b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-23T09:44:53.243895553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-23T09:44:53.248950376Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db96333a3%3A19db9635646%3A3d32df0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-23T09:44:56.012464311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a0d2eed16cfb5aad immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-cwev\"}" level=error ts=2026-04-23T09:44:56.015900212Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a0d2eed16cfb5aad/19db96ed326%3A19db96ef035%3A33aed80e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-cwev\"}" level=warn ts=2026-04-23T09:44:56.746177885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:44:57.034397643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-7e16cc4f 10.129.0.88:7946" level=debug ts=2026-04-23T09:44:57.088595403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-23T09:44:57.34584865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-23T09:44:57.350369742Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db962e775%3A19db9802088%3A905386ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-23T09:44:57.570996644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2da642124d384da8 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"konflux-ypno\"}" level=error ts=2026-04-23T09:44:57.574882574Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2da642124d384da8/19db96fff6b%3A19db97e209d%3A8d10409c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ypno\"}" level=info ts=2026-04-23T09:44:58.278223196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e010f61302c79b78 immediate=false num_chunks=1 total_comp="787 kB" avg_comp="787 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" full=1 labels="{stream=\"build-e2e-zkra\"}" level=error ts=2026-04-23T09:44:58.282497931Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e010f61302c79b78/19db96ff79b%3A19db9833f6d%3A3ddabfc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zkra\"}" level=info ts=2026-04-23T09:44:59.263379543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-23T09:44:59.266844165Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db9634c89%3A19db9634c8a%3A62f07f65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-23T09:44:59.429162486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="420 kB" avg_comp="420 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-23T09:44:59.432961932Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db962eb8a%3A19db9a07247%3A2ebb1595?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=info ts=2026-04-23T09:44:59.439137978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2936a11c98a836c immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="859 kB" avg_uncomp="859 kB" full=1 labels="{stream=\"stat-rep-sjil\"}" level=error ts=2026-04-23T09:44:59.442182545Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a2936a11c98a836c/19db96f540d%3A19db9769b57%3Af0d6f9f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-sjil\"}" level=info ts=2026-04-23T09:44:59.579811525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="484 kB" avg_comp="484 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-23T09:44:59.583641667Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db962e7a2%3A19db99e5cd6%3Ab5d3b01b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-23T09:44:59.795325917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-23T09:44:59.798678722Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db96f632b%3A19db970e04e%3A45b631d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-23T09:45:00.145091002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-23T09:45:00.148419613Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db96333a3%3A19db9635646%3A5ab093ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-23T09:45:00.52951445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="487 kB" avg_comp="487 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}" level=error ts=2026-04-23T09:45:00.533676529Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db962e8f5%3A19db9afb404%3Ab41c8433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}" level=info ts=2026-04-23T09:45:00.750378935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9f37ee7ca8074ee0 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-tqxo\"}" level=error ts=2026-04-23T09:45:00.753832092Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9f37ee7ca8074ee0/19db96f888c%3A19db9714bf8%3A484e613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-tqxo\"}" level=warn ts=2026-04-23T09:45:01.74563584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:45:02.463346129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-23T09:45:02.467856264Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db9634c89%3A19db9635646%3A62a82fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-23T09:45:05.529735607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-23T09:45:05.533423299Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db96333a3%3A19db9760eb7%3A84cd9a59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-23T09:45:06.302922555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="696 kB" avg_comp="696 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-23T09:45:06.307195869Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db962e8e4%3A19db993161d%3A918ee0bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=warn ts=2026-04-23T09:45:06.745319815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:45:07.631745334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-23T09:45:07.635920214Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db962eaf7%3A19db9946c27%3Aec3e3ab9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-23T09:45:08.929097366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-23T09:45:08.932498144Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db962e782%3A19db9635646%3Aecb163a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-23T09:45:11.746127382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-23T09:45:16.746180873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:45:18.363072481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56896" level=debug ts=2026-04-23T09:45:18.972540085Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-23T09:45:20.797996203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-23T09:45:20.842981968Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-23T09:45:20.843247451Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1 level=debug ts=2026-04-23T09:45:20.844739546Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads" level=debug ts=2026-04-23T09:45:20.844777828Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974374 n=1 level=debug ts=2026-04-23T09:45:20.844822838Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974374 n=1 level=info ts=2026-04-23T09:45:21.74545494Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-23T09:45:21.745487521Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-23T09:45:21.745497356Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-23T09:45:21.745523577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-23T09:45:26.745416163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:45:27.036437969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946" level=info ts=2026-04-23T09:45:30.165486121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e010f61302c79b78 immediate=false num_chunks=1 total_comp="787 kB" avg_comp="787 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" full=1 labels="{stream=\"build-e2e-zkra\"}" level=error ts=2026-04-23T09:45:30.172484643Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e010f61302c79b78/19db96ff79b%3A19db9833f6d%3A3ddabfc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zkra\"}" level=info ts=2026-04-23T09:45:30.900104662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="903 kB" avg_comp="903 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}" level=error ts=2026-04-23T09:45:30.90490168Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19db962fc91%3A19db9a3c322%3Af163b89c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}" level=warn ts=2026-04-23T09:45:31.746154988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:45:34.703081581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-23T09:45:34.709425282Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19db96333a3%3A19db9a6bba7%3Afdefc619?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-23T09:45:35.346344988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-23T09:45:35.353111663Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db96333a4%3A19db9a9e43d%3A88a81321?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-23T09:45:36.746010237Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:45:37.690300749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-23T09:45:37.69377197Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db962ea76%3A19db97c8e44%3A20dbb383?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-23T09:45:39.306811056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-23T09:45:39.31010862Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db962eb43%3A19db96333a5%3Adc6ca458?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-23T09:45:39.893168918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=52e28635a1e69389 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-bnlr\"}" level=error ts=2026-04-23T09:45:39.896652073Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/52e28635a1e69389/19db96f329c%3A19db9761604%3Accb48298?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-bnlr\"}" level=info ts=2026-04-23T09:45:41.539924085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-23T09:45:41.543440441Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db96333a3%3A19db9635646%3A5ab093ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=warn ts=2026-04-23T09:45:41.746097032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:45:42.52307531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-23T09:45:42.527465054Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db962e775%3A19db9802088%3A905386ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-23T09:45:43.259595399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-23T09:45:43.26312791Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db96333a4%3A19db96399de%3A777b031b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-23T09:45:44.66941906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=63fa18e885493035 immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="42 MB" avg_uncomp="10 MB" full=3 synced=1 labels="{stream=\"group-jsux\"}" level=error ts=2026-04-23T09:45:44.677506513Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/63fa18e885493035/19db96f3c9e%3A19db97f85d3%3Ac8c91f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsux\"}" level=debug ts=2026-04-23T09:45:45.291864375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-04-23T09:45:45.293562449Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-04-23T09:45:46.688451169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="487 kB" avg_comp="487 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}" level=error ts=2026-04-23T09:45:46.692467104Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db962e8f5%3A19db9afb404%3Ab41c8433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}" level=warn ts=2026-04-23T09:45:46.745754328Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:45:47.737363058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2936a11c98a836c immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="859 kB" avg_uncomp="859 kB" full=1 labels="{stream=\"stat-rep-sjil\"}" level=error ts=2026-04-23T09:45:47.741657471Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a2936a11c98a836c/19db96f540d%3A19db9769b57%3Af0d6f9f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-sjil\"}" level=debug ts=2026-04-23T09:45:47.900163257Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-23T09:45:48.247671854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="420 kB" avg_comp="420 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-23T09:45:48.251895645Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db962eb8a%3A19db9a07247%3A2ebb1595?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=info ts=2026-04-23T09:45:48.991845323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0a598b07aac1932a immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-mnwl\"}" level=error ts=2026-04-23T09:45:48.996588608Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a598b07aac1932a/19db96f7504%3A19db9830708%3A4cfb3855?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-mnwl\"}" level=info ts=2026-04-23T09:45:49.906916711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-23T09:45:49.910515065Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db96f632b%3A19db970e04e%3A45b631d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-23T09:45:51.745424724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-23T09:45:51.74545451Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-23T09:45:51.745463168Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-23T09:45:51.745470086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:45:51.860269331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-23T09:45:51.863904832Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db96333a3%3A19db9635646%3A3d32df0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-23T09:45:53.146164284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-23T09:45:53.149923155Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db9634c89%3A19db9635646%3A62a82fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-23T09:45:53.292309637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a0d2eed16cfb5aad immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-cwev\"}" level=error ts=2026-04-23T09:45:53.295828856Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a0d2eed16cfb5aad/19db96ed326%3A19db96ef035%3A33aed80e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-cwev\"}" level=info ts=2026-04-23T09:45:54.176336722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="696 kB" avg_comp="696 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-23T09:45:54.180565327Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db962e8e4%3A19db993161d%3A918ee0bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-23T09:45:54.475457815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-23T09:45:54.478636566Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db962e782%3A19db9635646%3Aecb163a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-23T09:45:55.615854858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-23T09:45:55.61925575Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db9634c89%3A19db9634c8a%3A62f07f65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-23T09:45:55.899138771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2da642124d384da8 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"konflux-ypno\"}" level=error ts=2026-04-23T09:45:55.90321212Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2da642124d384da8/19db96fff6b%3A19db97e209d%3A8d10409c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ypno\"}" level=info ts=2026-04-23T09:45:56.271494377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="484 kB" avg_comp="484 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-23T09:45:56.276054112Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db962e7a2%3A19db99e5cd6%3Ab5d3b01b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-23T09:45:56.745560669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:45:57.038885971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-7e16cc4f 10.129.0.88:7946" level=info ts=2026-04-23T09:45:58.117386167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9f37ee7ca8074ee0 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-tqxo\"}" level=error ts=2026-04-23T09:45:58.120981457Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9f37ee7ca8074ee0/19db96f888c%3A19db9714bf8%3A484e613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-tqxo\"}" level=info ts=2026-04-23T09:45:59.932170127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-23T09:45:59.936027386Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db962eaf7%3A19db9946c27%3Aec3e3ab9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=warn ts=2026-04-23T09:46:01.74588075Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:46:03.681145791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-23T09:46:03.686619113Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db96333a3%3A19db9760eb7%3A84cd9a59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=warn ts=2026-04-23T09:46:06.746165924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-23T09:46:11.745566595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:46:12.38227271Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 ts=2026-04-23T09:46:13.10427538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:47352" level=info ts=2026-04-23T09:46:16.479031769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-23T09:46:16.486539491Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db96333a4%3A19db9a9e43d%3A88a81321?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-23T09:46:16.570806108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-23T09:46:16.577741463Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19db96333a3%3A19db9a6bba7%3Afdefc619?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=warn ts=2026-04-23T09:46:16.745412844Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:46:18.568789762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e010f61302c79b78 immediate=false num_chunks=1 total_comp="787 kB" avg_comp="787 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" full=1 labels="{stream=\"build-e2e-zkra\"}" level=error ts=2026-04-23T09:46:18.575344697Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e010f61302c79b78/19db96ff79b%3A19db9833f6d%3A3ddabfc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zkra\"}" level=info ts=2026-04-23T09:46:20.842988882Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-23T09:46:20.843303594Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1 level=info ts=2026-04-23T09:46:21.746185827Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-23T09:46:21.746219856Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=warn ts=2026-04-23T09:46:21.746207456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:46:21.746228179Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-23T09:46:23.7707654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-23T09:46:23.896274797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-23T09:46:23.900786504Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db962e775%3A19db9802088%3A905386ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-23T09:46:24.158515617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-23T09:46:24.162044847Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db962eb43%3A19db96333a5%3Adc6ca458?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-23T09:46:26.746076113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:46:27.040279375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-7e16cc4f 10.129.0.88:7946" level=info ts=2026-04-23T09:46:27.395525833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="903 kB" avg_comp="903 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}" level=error ts=2026-04-23T09:46:27.400169317Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19db962fc91%3A19db9a3c322%3Af163b89c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}" level=info ts=2026-04-23T09:46:29.479875579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0a598b07aac1932a immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-mnwl\"}" level=error ts=2026-04-23T09:46:29.484713055Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a598b07aac1932a/19db96f7504%3A19db9830708%3A4cfb3855?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-mnwl\"}" level=info ts=2026-04-23T09:46:30.198886221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-23T09:46:30.202578175Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db96333a4%3A19db96399de%3A777b031b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-23T09:46:31.331797825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="420 kB" avg_comp="420 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-23T09:46:31.335813363Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db962eb8a%3A19db9a07247%3A2ebb1595?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=warn ts=2026-04-23T09:46:31.746106825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:46:33.699933172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-23T09:46:33.705899459Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db96333a3%3A19db9635646%3A5ab093ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-23T09:46:34.710189483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-23T09:46:34.713815846Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db962ea76%3A19db97c8e44%3A20dbb383?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-23T09:46:34.938556876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-23T09:46:34.942099941Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db96f632b%3A19db970e04e%3A45b631d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-23T09:46:35.862463503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=52e28635a1e69389 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-bnlr\"}" level=error ts=2026-04-23T09:46:35.866050634Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/52e28635a1e69389/19db96f329c%3A19db9761604%3Accb48298?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-bnlr\"}" level=warn ts=2026-04-23T09:46:36.746096517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:46:38.248400971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-23T09:46:38.251924496Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db962e782%3A19db9635646%3Aecb163a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-23T09:46:38.51856745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2da642124d384da8 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"konflux-ypno\"}" level=error ts=2026-04-23T09:46:38.522587043Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2da642124d384da8/19db96fff6b%3A19db97e209d%3A8d10409c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ypno\"}" level=info ts=2026-04-23T09:46:40.682594155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=63fa18e885493035 immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="42 MB" avg_uncomp="10 MB" full=3 synced=1 labels="{stream=\"group-jsux\"}" level=error ts=2026-04-23T09:46:40.690873861Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/63fa18e885493035/19db96f3c9e%3A19db97f85d3%3Ac8c91f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsux\"}" level=warn ts=2026-04-23T09:46:41.74567153Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:46:42.659813956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2936a11c98a836c immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="859 kB" avg_uncomp="859 kB" full=1 labels="{stream=\"stat-rep-sjil\"}" level=error ts=2026-04-23T09:46:42.663727846Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a2936a11c98a836c/19db96f540d%3A19db9769b57%3Af0d6f9f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-sjil\"}" level=info ts=2026-04-23T09:46:43.030742823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="487 kB" avg_comp="487 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}" level=error ts=2026-04-23T09:46:43.034867323Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db962e8f5%3A19db9afb404%3Ab41c8433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}" ts=2026-04-23T09:46:43.105482249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:53768" level=info ts=2026-04-23T09:46:43.148234724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-23T09:46:43.152541946Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db96333a3%3A19db9635646%3A3d32df0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-23T09:46:43.924738737Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-23T09:46:43.958892728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-23T09:46:43.962447853Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db9634c89%3A19db9634c8a%3A62f07f65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-23T09:46:44.074845207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="696 kB" avg_comp="696 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-23T09:46:44.079363553Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db962e8e4%3A19db993161d%3A918ee0bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-23T09:46:45.665469593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-23T09:46:45.669484623Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db9634c89%3A19db9635646%3A62a82fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-23T09:46:45.752647099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-23T09:46:45.756831908Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db962eaf7%3A19db9946c27%3Aec3e3ab9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-23T09:46:46.561549236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="484 kB" avg_comp="484 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-23T09:46:46.565652547Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db962e7a2%3A19db99e5cd6%3Ab5d3b01b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-23T09:46:46.745300444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:46:47.671754333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a0d2eed16cfb5aad immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-cwev\"}" level=error ts=2026-04-23T09:46:47.675349118Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a0d2eed16cfb5aad/19db96ed326%3A19db96ef035%3A33aed80e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-cwev\"}" level=info ts=2026-04-23T09:46:51.745627761Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-23T09:46:51.745646807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:46:51.74566468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-23T09:46:51.745676784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-23T09:46:53.410795891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9f37ee7ca8074ee0 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-tqxo\"}" level=error ts=2026-04-23T09:46:53.41454836Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9f37ee7ca8074ee0/19db96f888c%3A19db9714bf8%3A484e613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-tqxo\"}" level=info ts=2026-04-23T09:46:53.670395627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-23T09:46:53.674011437Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db96333a3%3A19db9760eb7%3A84cd9a59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=warn ts=2026-04-23T09:46:56.745913123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:46:57.042043876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946" level=info ts=2026-04-23T09:46:58.061634389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-23T09:46:58.068730366Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db96333a4%3A19db9a9e43d%3A88a81321?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-23T09:47:00.721220604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-23T09:47:00.728318668Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19db96333a3%3A19db9a6bba7%3Afdefc619?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=warn ts=2026-04-23T09:47:01.745433286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:47:05.305850575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-23T09:47:05.312883357Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db962e775%3A19db9802088%3A905386ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}" level=warn ts=2026-04-23T09:47:06.746201522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:47:11.134378492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-23T09:47:11.138291737Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db962eb43%3A19db96333a5%3Adc6ca458?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-23T09:47:11.745950871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:47:12.813140701Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-23T09:47:14.087309453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e010f61302c79b78 immediate=false num_chunks=1 total_comp="787 kB" avg_comp="787 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" full=1 labels="{stream=\"build-e2e-zkra\"}" level=error ts=2026-04-23T09:47:14.092024049Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e010f61302c79b78/19db96ff79b%3A19db9833f6d%3A3ddabfc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zkra\"}" level=info ts=2026-04-23T09:47:14.677946056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0a598b07aac1932a immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-mnwl\"}" level=error ts=2026-04-23T09:47:14.682875929Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a598b07aac1932a/19db96f7504%3A19db9830708%3A4cfb3855?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-mnwl\"}" level=warn ts=2026-04-23T09:47:16.746176912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:47:17.979260845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-23T09:47:17.9827593Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db96333a3%3A19db9635646%3A5ab093ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-23T09:47:18.346874713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-23T09:47:18.350472787Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db96333a4%3A19db96399de%3A777b031b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" ts=2026-04-23T09:47:18.654877695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46656" level=debug ts=2026-04-23T09:47:20.842759013Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1 level=info ts=2026-04-23T09:47:20.843442907Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-23T09:47:21.745848949Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-23T09:47:21.745878118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-23T09:47:21.745886431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-23T09:47:21.745906425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:47:23.289158875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="903 kB" avg_comp="903 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}" level=error ts=2026-04-23T09:47:23.296113913Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19db962fc91%3A19db9a3c322%3Af163b89c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}" level=info ts=2026-04-23T09:47:23.444663373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=52e28635a1e69389 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-bnlr\"}" level=error ts=2026-04-23T09:47:23.448338902Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/52e28635a1e69389/19db96f329c%3A19db9761604%3Accb48298?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-bnlr\"}" level=info ts=2026-04-23T09:47:23.818780152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="487 kB" avg_comp="487 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}" level=error ts=2026-04-23T09:47:23.822803675Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db962e8f5%3A19db9afb404%3Ab41c8433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}" level=info ts=2026-04-23T09:47:23.979508675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-23T09:47:23.983043338Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db962ea76%3A19db97c8e44%3A20dbb383?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=warn ts=2026-04-23T09:47:26.745366238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:47:27.044439371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-7e16cc4f 10.129.0.88:7946" level=debug ts=2026-04-23T09:47:27.141220219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-23T09:47:28.771219489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="696 kB" avg_comp="696 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-23T09:47:28.775638841Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db962e8e4%3A19db993161d%3A918ee0bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-23T09:47:29.405046075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="420 kB" avg_comp="420 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-23T09:47:29.409040106Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db962eb8a%3A19db9a07247%3A2ebb1595?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=info ts=2026-04-23T09:47:31.199281303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-23T09:47:31.202697521Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db96333a3%3A19db9635646%3A3d32df0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-23T09:47:31.745417724Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:47:31.924649382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-23T09:47:31.928010128Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db962e782%3A19db9635646%3Aecb163a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-23T09:47:32.160508412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-23T09:47:32.163985715Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db9634c89%3A19db9634c8a%3A62f07f65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-23T09:47:32.371581466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-23T09:47:32.374969516Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db96f632b%3A19db970e04e%3A45b631d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-23T09:47:34.250657051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-23T09:47:34.254158543Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db96333a3%3A19db9760eb7%3A84cd9a59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-23T09:47:34.799119438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2936a11c98a836c immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="859 kB" avg_uncomp="859 kB" full=1 labels="{stream=\"stat-rep-sjil\"}" level=error ts=2026-04-23T09:47:34.802717388Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a2936a11c98a836c/19db96f540d%3A19db9769b57%3Af0d6f9f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-sjil\"}" level=info ts=2026-04-23T09:47:34.854401034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a0d2eed16cfb5aad immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-cwev\"}" level=error ts=2026-04-23T09:47:34.857751539Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a0d2eed16cfb5aad/19db96ed326%3A19db96ef035%3A33aed80e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-cwev\"}" level=info ts=2026-04-23T09:47:35.737239605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=63fa18e885493035 immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="42 MB" avg_uncomp="10 MB" full=3 synced=1 labels="{stream=\"group-jsux\"}" level=error ts=2026-04-23T09:47:35.744963971Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/63fa18e885493035/19db96f3c9e%3A19db97f85d3%3Ac8c91f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsux\"}" level=info ts=2026-04-23T09:47:36.09489324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="484 kB" avg_comp="484 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-23T09:47:36.098757204Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db962e7a2%3A19db99e5cd6%3Ab5d3b01b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-23T09:47:36.196021702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2da642124d384da8 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"konflux-ypno\"}" level=error ts=2026-04-23T09:47:36.199872217Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2da642124d384da8/19db96fff6b%3A19db97e209d%3A8d10409c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ypno\"}" level=warn ts=2026-04-23T09:47:36.745638716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:47:40.149941224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-23T09:47:40.189967111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-23T09:47:40.193793755Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db962eaf7%3A19db9946c27%3Aec3e3ab9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-23T09:47:41.454916434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9f37ee7ca8074ee0 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-tqxo\"}" level=error ts=2026-04-23T09:47:41.458395374Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9f37ee7ca8074ee0/19db96f888c%3A19db9714bf8%3A484e613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-tqxo\"}" level=warn ts=2026-04-23T09:47:41.746220428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:47:43.751997369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-23T09:47:43.755473355Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db9634c89%3A19db9635646%3A62a82fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-23T09:47:46.745743195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:47:48.377931927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:33642" ts=2026-04-23T09:47:48.65811639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36994" level=info ts=2026-04-23T09:47:51.74618204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-23T09:47:51.746189311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:47:51.746217902Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-23T09:47:51.746228468Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-23T09:47:52.114361974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-23T09:47:52.119404318Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db962e775%3A19db9802088%3A905386ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}" level=info ts=2026-04-23T09:47:55.384839987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-23T09:47:55.398110447Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db96333a4%3A19db9a9e43d%3A88a81321?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-23T09:47:56.745680536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:47:57.046559565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946" level=info ts=2026-04-23T09:48:00.372240252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-23T09:48:00.379212135Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19db96333a3%3A19db9a6bba7%3Afdefc619?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=warn ts=2026-04-23T09:48:01.746198957Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:48:03.295370194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0a598b07aac1932a immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-mnwl\"}" level=error ts=2026-04-23T09:48:03.300501192Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a598b07aac1932a/19db96f7504%3A19db9830708%3A4cfb3855?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-mnwl\"}" level=info ts=2026-04-23T09:48:03.362747881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="903 kB" avg_comp="903 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}" level=error ts=2026-04-23T09:48:03.367273015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19db962fc91%3A19db9a3c322%3Af163b89c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}" level=info ts=2026-04-23T09:48:03.467731117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e010f61302c79b78 immediate=false num_chunks=1 total_comp="787 kB" avg_comp="787 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" full=1 labels="{stream=\"build-e2e-zkra\"}" level=error ts=2026-04-23T09:48:03.472302149Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e010f61302c79b78/19db96ff79b%3A19db9833f6d%3A3ddabfc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zkra\"}" level=info ts=2026-04-23T09:48:04.246747082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="487 kB" avg_comp="487 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}" level=error ts=2026-04-23T09:48:04.25056482Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db962e8f5%3A19db9afb404%3Ab41c8433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}" level=debug ts=2026-04-23T09:48:06.099988767Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-23T09:48:06.74605529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:48:06.9413566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-23T09:48:06.94839932Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db962eb43%3A19db96333a5%3Adc6ca458?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-23T09:48:08.747516505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-23T09:48:08.750951397Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db96333a3%3A19db9635646%3A5ab093ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=info ts=2026-04-23T09:48:09.573652128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=52e28635a1e69389 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-bnlr\"}" level=error ts=2026-04-23T09:48:09.577144647Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/52e28635a1e69389/19db96f329c%3A19db9761604%3Accb48298?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-bnlr\"}" level=info ts=2026-04-23T09:48:11.638221633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-23T09:48:11.641985965Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db96333a4%3A19db96399de%3A777b031b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-23T09:48:11.745946674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:48:13.112631244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:57530" level=info ts=2026-04-23T09:48:13.799276538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="420 kB" avg_comp="420 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-23T09:48:13.803487504Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db962eb8a%3A19db9a07247%3A2ebb1595?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=info ts=2026-04-23T09:48:14.623925902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="696 kB" avg_comp="696 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-23T09:48:14.628456363Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db962e8e4%3A19db993161d%3A918ee0bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=warn ts=2026-04-23T09:48:16.746223982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:48:19.151808996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-23T09:48:19.15557155Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db96333a3%3A19db9760eb7%3A84cd9a59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=info ts=2026-04-23T09:48:19.859636674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="484 kB" avg_comp="484 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-23T09:48:19.863706609Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db962e7a2%3A19db99e5cd6%3Ab5d3b01b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-23T09:48:20.670873245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-23T09:48:20.674350722Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db962ea76%3A19db97c8e44%3A20dbb383?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-23T09:48:20.842771802Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1 level=info ts=2026-04-23T09:48:20.843124271Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-23T09:48:21.050320631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-23T09:48:21.05444606Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db962eaf7%3A19db9946c27%3Aec3e3ab9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-23T09:48:21.745354276Z caller=checkpoint.go:612 msg="starting checkpoint" level=info ts=2026-04-23T09:48:21.745373588Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-23T09:48:21.745377035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:48:21.745398023Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-23T09:48:21.74540695Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-23T09:48:21.745538265Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000021 level=debug ts=2026-04-23T09:48:21.745761696Z caller=checkpoint.go:378 msg="writing series" size="60 kB" level=info ts=2026-04-23T09:48:23.113878283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9f37ee7ca8074ee0 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-tqxo\"}" level=error ts=2026-04-23T09:48:23.117309346Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9f37ee7ca8074ee0/19db96f888c%3A19db9714bf8%3A484e613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-tqxo\"}" level=info ts=2026-04-23T09:48:23.977973833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-23T09:48:23.981718193Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db9634c89%3A19db9634c8a%3A62f07f65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-23T09:48:24.390819532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=63fa18e885493035 immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="42 MB" avg_uncomp="10 MB" full=3 synced=1 labels="{stream=\"group-jsux\"}" level=error ts=2026-04-23T09:48:24.398424972Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/63fa18e885493035/19db96f3c9e%3A19db97f85d3%3Ac8c91f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsux\"}" level=debug ts=2026-04-23T09:48:25.07996035Z caller=checkpoint.go:378 msg="writing series" size="187 kB" level=info ts=2026-04-23T09:48:26.306239212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-23T09:48:26.31089052Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db962e782%3A19db9635646%3Aecb163a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-23T09:48:26.746154292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:48:27.047942654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-7e16cc4f 10.129.0.88:7946" level=info ts=2026-04-23T09:48:27.341419662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-23T09:48:27.34483883Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db96333a3%3A19db9635646%3A3d32df0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-23T09:48:28.412822021Z caller=checkpoint.go:378 msg="writing series" size="271 kB" level=info ts=2026-04-23T09:48:31.192993804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2936a11c98a836c immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="859 kB" avg_uncomp="859 kB" full=1 labels="{stream=\"stat-rep-sjil\"}" level=error ts=2026-04-23T09:48:31.197266088Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a2936a11c98a836c/19db96f540d%3A19db9769b57%3Af0d6f9f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-sjil\"}" level=warn ts=2026-04-23T09:48:31.745873616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:48:31.745915379Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB" level=info ts=2026-04-23T09:48:31.881764672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-23T09:48:31.885495109Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db96f632b%3A19db970e04e%3A45b631d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-23T09:48:32.164242689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a0d2eed16cfb5aad immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-cwev\"}" level=error ts=2026-04-23T09:48:32.167871657Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a0d2eed16cfb5aad/19db96ed326%3A19db96ef035%3A33aed80e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-cwev\"}" level=info ts=2026-04-23T09:48:34.421448877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2da642124d384da8 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"konflux-ypno\"}" level=error ts=2026-04-23T09:48:34.425251319Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2da642124d384da8/19db96fff6b%3A19db97e209d%3A8d10409c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ypno\"}" level=info ts=2026-04-23T09:48:34.880621403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-23T09:48:34.884263838Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db9634c89%3A19db9635646%3A62a82fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-23T09:48:35.079785448Z caller=checkpoint.go:378 msg="writing series" size="200 kB" level=debug ts=2026-04-23T09:48:35.787912872Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-23T09:48:36.745636692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:48:38.412694406Z caller=checkpoint.go:378 msg="writing series" size="127 kB" level=debug ts=2026-04-23T09:48:41.518780072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-04-23T09:48:41.745464518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:48:41.746794658Z caller=checkpoint.go:378 msg="writing series" size="323 kB" level=debug ts=2026-04-23T09:48:41.746811103Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7 level=info ts=2026-04-23T09:48:42.529854521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}" level=error ts=2026-04-23T09:48:42.534416621Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db962e775%3A19db9802088%3A905386ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}" ts=2026-04-23T09:48:43.114122646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55278" level=debug ts=2026-04-23T09:48:45.080857264Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB" level=debug ts=2026-04-23T09:48:45.080881123Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=1 level=info ts=2026-04-23T09:48:45.814728948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-23T09:48:45.822248154Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db96333a4%3A19db9a9e43d%3A88a81321?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-23T09:48:46.745583776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:48:48.414516791Z caller=checkpoint.go:378 msg="writing series" size="2.7 MB" level=debug ts=2026-04-23T09:48:48.414541511Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=1 level=info ts=2026-04-23T09:48:51.745820007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-23T09:48:51.745848349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=debug ts=2026-04-23T09:48:51.745838657Z caller=checkpoint.go:378 msg="writing series" size="16 kB" level=info ts=2026-04-23T09:48:51.745856007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-23T09:48:51.745869557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:48:52.713987063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="903 kB" avg_comp="903 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}" level=error ts=2026-04-23T09:48:52.718948846Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19db962fc91%3A19db9a3c322%3Af163b89c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}" level=info ts=2026-04-23T09:48:53.073240423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-23T09:48:53.079875682Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19db96333a3%3A19db9a6bba7%3Afdefc619?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-23T09:48:53.531976772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e010f61302c79b78 immediate=false num_chunks=1 total_comp="787 kB" avg_comp="787 kB" total_uncomp="5.2 MB" avg_uncomp="5.2 MB" full=1 labels="{stream=\"build-e2e-zkra\"}" level=error ts=2026-04-23T09:48:53.537041077Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e010f61302c79b78/19db96ff79b%3A19db9833f6d%3A3ddabfc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zkra\"}" level=info ts=2026-04-23T09:48:54.073570696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0a598b07aac1932a immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-mnwl\"}" level=error ts=2026-04-23T09:48:54.07829316Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a598b07aac1932a/19db96f7504%3A19db9830708%3A4cfb3855?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-mnwl\"}" level=debug ts=2026-04-23T09:48:55.079697726Z caller=checkpoint.go:378 msg="writing series" size="99 kB" level=warn ts=2026-04-23T09:48:56.74579731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:48:57.050245399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946" level=info ts=2026-04-23T09:48:57.411855397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-23T09:48:57.416043013Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db96333a4%3A19db96399de%3A777b031b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-23T09:48:58.309783097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-23T09:48:58.313600799Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db962eb43%3A19db96333a5%3Adc6ca458?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-23T09:48:58.336688388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="487 kB" avg_comp="487 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}" level=error ts=2026-04-23T09:48:58.340517477Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db962e8f5%3A19db9afb404%3Ab41c8433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}" level=debug ts=2026-04-23T09:48:58.415897078Z caller=checkpoint.go:378 msg="writing series" size="4.7 MB" level=debug ts=2026-04-23T09:48:58.415919939Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.8 MB" series=3 level=info ts=2026-04-23T09:49:01.667793545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="420 kB" avg_comp="420 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}" level=error ts=2026-04-23T09:49:01.671837899Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db962eb8a%3A19db9a07247%3A2ebb1595?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}" level=info ts=2026-04-23T09:49:01.713912589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}" level=error ts=2026-04-23T09:49:01.717293653Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db96333a3%3A19db9635646%3A5ab093ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}" level=warn ts=2026-04-23T09:49:01.745575333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:49:01.746631567Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB" level=info ts=2026-04-23T09:49:01.864456744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="696 kB" avg_comp="696 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-23T09:49:01.868662974Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db962e8e4%3A19db993161d%3A918ee0bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=debug ts=2026-04-23T09:49:04.845954066Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-23T09:49:04.904007913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=52e28635a1e69389 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-bnlr\"}" level=error ts=2026-04-23T09:49:04.908035359Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/52e28635a1e69389/19db96f329c%3A19db9761604%3Accb48298?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-bnlr\"}" level=info ts=2026-04-23T09:49:05.017289404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="418 kB" avg_comp="418 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-23T09:49:05.0216627Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db962eaf7%3A19db9946c27%3Aec3e3ab9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=debug ts=2026-04-23T09:49:05.080241584Z caller=checkpoint.go:378 msg="writing series" size="682 kB" level=warn ts=2026-04-23T09:49:06.745712898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:49:08.415128088Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB" level=debug ts=2026-04-23T09:49:08.415151948Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 MB" series=3 level=info ts=2026-04-23T09:49:08.620827574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=63fa18e885493035 immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="42 MB" avg_uncomp="10 MB" full=3 synced=1 labels="{stream=\"group-jsux\"}" level=error ts=2026-04-23T09:49:08.628572254Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/63fa18e885493035/19db96f3c9e%3A19db97f85d3%3Ac8c91f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsux\"}" level=info ts=2026-04-23T09:49:10.768923343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9f37ee7ca8074ee0 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-tqxo\"}" level=info ts=2026-04-23T09:49:10.770131656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-23T09:49:10.77267399Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9f37ee7ca8074ee0/19db96f888c%3A19db9714bf8%3A484e613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-tqxo\"}" level=error ts=2026-04-23T09:49:10.772680183Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db96333a3%3A19db9635646%3A3d32df0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=warn ts=2026-04-23T09:49:11.745668677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:49:11.745774629Z caller=checkpoint.go:378 msg="writing series" size="116 kB" level=info ts=2026-04-23T09:49:13.573993443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="484 kB" avg_comp="484 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-23T09:49:13.578134168Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db962e7a2%3A19db99e5cd6%3Ab5d3b01b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-23T09:49:15.07912072Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB" level=info ts=2026-04-23T09:49:15.246651924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-23T09:49:15.250254357Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db962e782%3A19db9635646%3Aecb163a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-23T09:49:15.870633664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a0d2eed16cfb5aad immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-cwev\"}" level=error ts=2026-04-23T09:49:15.874481578Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a0d2eed16cfb5aad/19db96ed326%3A19db96ef035%3A33aed80e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-cwev\"}" level=info ts=2026-04-23T09:49:16.052037272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}" level=error ts=2026-04-23T09:49:16.055368147Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db96333a3%3A19db9760eb7%3A84cd9a59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}" level=warn ts=2026-04-23T09:49:16.745554162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-23T09:49:17.18614845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-23T09:49:17.189543189Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db96f632b%3A19db970e04e%3A45b631d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-23T09:49:18.32574165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-23T09:49:18.329413829Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db962ea76%3A19db97c8e44%3A20dbb383?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-23T09:49:18.425396127Z caller=checkpoint.go:378 msg="writing series" size="12 MB" level=debug ts=2026-04-23T09:49:18.425421138Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=3 level=info ts=2026-04-23T09:49:18.568335085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-23T09:49:18.571629314Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db9634c89%3A19db9634c8a%3A62f07f65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-23T09:49:18.632712134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2da642124d384da8 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"konflux-ypno\"}" level=error ts=2026-04-23T09:49:18.636639527Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2da642124d384da8/19db96fff6b%3A19db97e209d%3A8d10409c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ypno\"}" level=info ts=2026-04-23T09:49:20.090806536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-23T09:49:20.094245242Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db9634c89%3A19db9635646%3A62a82fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-23T09:49:20.842803654Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1 level=info ts=2026-04-23T09:49:20.843848702Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-23T09:49:21.745638675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-23T09:49:21.745671436Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-23T09:49:21.745680065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-23T09:49:21.745702232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-23T09:49:21.745922907Z caller=checkpoint.go:378 msg="writing series" size="363 kB" level=debug ts=2026-04-23T09:49:25.079361286Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=info ts=2026-04-23T09:49:26.269423979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2936a11c98a836c immediate=false num_chunks=1 total_comp="190 kB" avg_comp="190 kB" total_uncomp="859 kB" avg_uncomp="859 kB" full=1 labels="{stream=\"stat-rep-sjil\"}" level=error ts=2026-04-23T09:49:26.273196552Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a2936a11c98a836c/19db96f540d%3A19db9769b57%3Af0d6f9f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-sjil\"}" level=warn ts=2026-04-23T09:49:26.745677283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-23T09:49:27.052896906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"