level=info ts=2026-04-25T00:11:06.987270555Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)" level=info ts=2026-04-25T00:11:06.987303887Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml level=debug ts=2026-04-25T00:11:06.987646178Z caller=modules.go:1984 msg="initializing ingester query tags interceptors" level=info ts=2026-04-25T00:11:06.988293054Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095 level=debug ts=2026-04-25T00:11:06.98867859Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-25T00:11:06.988755276Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.75 inf=eth0 level=debug ts=2026-04-25T00:11:06.991495692Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.49:11211 count=1 level=info ts=2026-04-25T00:11:06.992712243Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode" level=info ts=2026-04-25T00:11:06.992891418Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-25T00:11:06.993388908Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0 level=debug ts=2026-04-25T00:11:06.993405933Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-25T00:11:06.993404409Z level=debug ts=2026-04-25T00:11:06.993479239Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads" level=info ts=2026-04-25T00:11:06.993494457Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0 level=info ts=2026-04-25T00:11:06.993508344Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1 level=debug ts=2026-04-25T00:11:06.993512449Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-04-25T00:11:06.993511976Z level=debug ts=2026-04-25T00:11:06.993516712Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads" level=info ts=2026-04-25T00:11:06.994266735Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-8d775ed8 ts=2026-04-25T00:11:06.994317556Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired" level=debug ts=2026-04-25T00:11:06.994333222Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.75 advertisePort=7946 level=debug ts=2026-04-25T00:11:06.994354375Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.75 advertisePort=7946 level=info ts=2026-04-25T00:11:06.994537436Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4 level=debug ts=2026-04-25T00:11:06.995170657Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-25T00:11:06.995276287Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.75 inf=eth0 ts=2026-04-25T00:11:07.001114774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.130.0.48:7946" ts=2026-04-25T00:11:07.00297272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.129.0.74:7946" ts=2026-04-25T00:11:07.004286937Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.73:7946: dial tcp 10.129.0.73:7946: connect: connection refused" ts=2026-04-25T00:11:07.005059609Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.72:7946: dial tcp 10.129.0.72:7946: connect: connection refused" ts=2026-04-25T00:11:07.005842925Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.71:7946: dial tcp 10.129.0.71:7946: connect: connection refused" ts=2026-04-25T00:11:07.006506862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.129.0.68:7946" level=info ts=2026-04-25T00:11:07.007144188Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=12.609071ms level=info ts=2026-04-25T00:11:07.007172407Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local level=info ts=2026-04-25T00:11:07.00915661Z caller=module_service.go:82 msg=starting module=runtime-config level=debug ts=2026-04-25T00:11:07.009180199Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server level=debug ts=2026-04-25T00:11:07.009182015Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server level=debug ts=2026-04-25T00:11:07.009198107Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv level=debug ts=2026-04-25T00:11:07.009308644Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv level=debug ts=2026-04-25T00:11:07.009304898Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring level=info ts=2026-04-25T00:11:07.009319366Z caller=module_service.go:82 msg=starting module=server level=info ts=2026-04-25T00:11:07.009363149Z caller=module_service.go:82 msg=starting module=memberlist-kv level=debug ts=2026-04-25T00:11:07.009381264Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv level=debug ts=2026-04-25T00:11:07.009387736Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config level=debug ts=2026-04-25T00:11:07.009392255Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config level=debug ts=2026-04-25T00:11:07.009397392Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring level=debug ts=2026-04-25T00:11:07.009599327Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server level=info ts=2026-04-25T00:11:07.009610621Z caller=module_service.go:82 msg=starting module=ring level=debug ts=2026-04-25T00:11:07.009739592Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config level=info ts=2026-04-25T00:11:07.009685164Z caller=module_service.go:82 msg=starting module=store level=debug ts=2026-04-25T00:11:07.009757561Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server level=info ts=2026-04-25T00:11:07.009763152Z caller=module_service.go:82 msg=starting module=analytics level=debug ts=2026-04-25T00:11:07.009794686Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config level=debug ts=2026-04-25T00:11:07.009803216Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server level=debug ts=2026-04-25T00:11:07.009806784Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store level=debug ts=2026-04-25T00:11:07.009810032Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics level=debug ts=2026-04-25T00:11:07.009812724Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv level=info ts=2026-04-25T00:11:07.009815238Z caller=module_service.go:82 msg=starting module=ingester level=info ts=2026-04-25T00:11:07.00982997Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint" ts=2026-04-25T00:11:07.010255918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.130.0.48:7946" ts=2026-04-25T00:11:07.011107126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.129.0.74:7946" ts=2026-04-25T00:11:07.011675924Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.73:7946: dial tcp 10.129.0.73:7946: connect: connection refused" ts=2026-04-25T00:11:07.011782961Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.72:7946: dial tcp 10.129.0.72:7946: connect: connection refused" ts=2026-04-25T00:11:07.011895272Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.71:7946: dial tcp 10.129.0.71:7946: connect: connection refused" ts=2026-04-25T00:11:07.012034007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.129.0.68:7946" level=info ts=2026-04-25T00:11:07.012753977Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=5.576224ms level=info ts=2026-04-25T00:11:07.263658576Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=253.829349ms errors=false level=info ts=2026-04-25T00:11:07.263688172Z caller=ingester.go:587 component=ingester msg="recovering from WAL" level=info ts=2026-04-25T00:11:07.283396625Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=273.567658ms errors=false level=info ts=2026-04-25T00:11:07.283455071Z caller=ingester.go:551 component=ingester msg="closing recoverer" level=info ts=2026-04-25T00:11:07.283489327Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=273.661001ms level=info ts=2026-04-25T00:11:07.283511825Z caller=wal.go:158 msg=started component=wal level=info ts=2026-04-25T00:11:07.283569768Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty" level=info ts=2026-04-25T00:11:07.283570053Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=11.475092275s level=info ts=2026-04-25T00:11:07.283612034Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z level=debug ts=2026-04-25T00:11:07.283644986Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester level=info ts=2026-04-25T00:11:07.283672069Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat" level=info ts=2026-04-25T00:11:07.283693794Z caller=loki.go:599 msg="Loki started" startup_time=352.910871ms level=warn ts=2026-04-25T00:11:12.284858872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-25T00:11:17.284714163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-25T00:11:22.284459698Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:11:24.854631858Z caller=logging.go:137 msg="GET /ready (503) 98.062µs" level=warn ts=2026-04-25T00:11:27.283818644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:11:28.050174308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:48484" ts=2026-04-25T00:11:29.939536067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39278" level=warn ts=2026-04-25T00:11:32.284622843Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:11:34.852828293Z caller=logging.go:137 msg="GET /ready (503) 74.668µs" level=info ts=2026-04-25T00:11:37.283777901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=info ts=2026-04-25T00:11:37.283808672Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership" level=info ts=2026-04-25T00:11:37.283830494Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000 level=info ts=2026-04-25T00:11:37.283840434Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:11:37.283842905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-25T00:11:42.283978671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-25T00:11:47.284300564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-25T00:11:52.284621353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:11:54.058546899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="799 B" avg_comp="799 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:11:54.058544207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="759 B" avg_comp="759 B" total_uncomp="24 kB" avg_uncomp="24 kB" idle=1 labels="{stream=\"chains-e2e-kszx\"}" level=info ts=2026-04-25T00:11:54.058565806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=info ts=2026-04-25T00:11:54.058588313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:11:54.058602627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=info ts=2026-04-25T00:11:54.05860246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:11:54.05861612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:11:54.058742737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:11:54.058739094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=info ts=2026-04-25T00:11:54.058752234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=info ts=2026-04-25T00:11:54.058759694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=info ts=2026-04-25T00:11:54.058762951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:11:54.058793708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=info ts=2026-04-25T00:11:54.058902799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:11:54.058912595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:11:54.058975364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:11:54.058989032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=info ts=2026-04-25T00:11:54.05910107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:11:54.059336052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=debug ts=2026-04-25T00:11:54.05938305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=error ts=2026-04-25T00:11:54.064321135Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:11:54.064368313Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:11:54.064363771Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:11:54.064347484Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:11:54.064403119Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=error ts=2026-04-25T00:11:54.064338083Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:11:54.0644143Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:11:54.064434678Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:11:54.064341519Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:11:54.064342166Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:11:54.064457139Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:11:54.06433916Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=error ts=2026-04-25T00:11:54.064321071Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=error ts=2026-04-25T00:11:54.064321245Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:11:54.066183765Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=error ts=2026-04-25T00:11:54.066187902Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:11:54.066211957Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:11:54.066217262Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:11:54.06621729Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:11:57.283802838Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:11:58.052501245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:34430" ts=2026-04-25T00:12:00.036896162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-adc48d01 10.129.0.74:7946" level=warn ts=2026-04-25T00:12:02.284588648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:02.989501977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-25T00:12:04.937359041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:12:04.941100525Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:12:05.287746337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:12:05.290656015Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=info ts=2026-04-25T00:12:05.330722123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:12:05.334665385Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-25T00:12:05.660304638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:12:05.663138753Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:12:06.165561367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:12:06.168947932Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=info ts=2026-04-25T00:12:06.270344862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:12:06.275587118Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" level=info ts=2026-04-25T00:12:06.349789108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:12:06.355453605Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:12:06.530945224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:12:06.536393595Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:12:06.917386633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:12:06.922062869Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-25T00:12:06.927135338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:12:06.932537284Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:12:06.992589531Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.49:11211 count=1 level=info ts=2026-04-25T00:12:06.993641184Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-25T00:12:07.284450592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:12:07.284480597Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:12:07.284489335Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:12:07.284510545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:12:09.028541513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=error ts=2026-04-25T00:12:09.035090924Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=info ts=2026-04-25T00:12:09.321897611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:12:09.324957374Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:12:10.237551728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:12:10.240858147Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:12:10.379072118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:12:10.382714114Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:12:11.810331446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:12:11.813733758Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:12:11.868786572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:12:11.872316383Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-25T00:12:11.965572871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:12:11.968634772Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:12:12.061473901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:12:12.06449394Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=info ts=2026-04-25T00:12:12.100698815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:12:12.10373524Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:12:12.284576399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-25T00:12:17.284244022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:19.824612919Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-04-25T00:12:22.283998581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:12:24.48894947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:44428" level=info ts=2026-04-25T00:12:25.843001191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:12:25.847190133Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-25T00:12:26.539111871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:12:26.544658093Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:12:27.102704925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:12:27.108834767Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:12:27.284269185Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:27.714212079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:12:29.05576572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:12:29.060838888Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-25T00:12:29.068905987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:12:29.073919848Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" ts=2026-04-25T00:12:30.03817349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946" level=info ts=2026-04-25T00:12:31.242382014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:12:31.245925993Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=warn ts=2026-04-25T00:12:32.284009313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:12:33.502574489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:12:33.505811907Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:12:33.765197175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:12:33.768721975Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:12:34.996985857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=error ts=2026-04-25T00:12:35.005715369Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=info ts=2026-04-25T00:12:35.440916144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:12:35.444171942Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=info ts=2026-04-25T00:12:35.77193005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:12:35.778062931Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:12:36.423223496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:12:36.426010404Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:12:37.284230014Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:12:37.284262756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:12:37.284272008Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:12:37.284271755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:37.392278456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-25T00:12:37.658751216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:12:37.661593579Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:12:37.913070247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:12:37.916208494Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:12:42.284087127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:12:42.446474241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:12:42.449676999Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:12:42.736106764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:12:42.739821569Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-25T00:12:46.182451612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:12:46.18570084Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=info ts=2026-04-25T00:12:46.444534399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:12:46.447695794Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:12:47.284697146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:12:48.060541983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:12:48.063658024Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=warn ts=2026-04-25T00:12:52.28383693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:12:54.536492299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:12:57.283996088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:12:59.943808918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42026" ts=2026-04-25T00:13:00.04044456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946" level=debug ts=2026-04-25T00:13:00.990980351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-04-25T00:13:02.284096419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:06.993007588Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:13:06.993288306Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.49:11211 count=1 level=info ts=2026-04-25T00:13:07.28421911Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:13:07.284236357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:13:07.284258421Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:13:07.284267857Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-25T00:13:08.821692723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:13:08.828885358Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:13:10.417541718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:13:10.423255886Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" level=warn ts=2026-04-25T00:13:12.284643853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:16.717657812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:13:16.720446453Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:13:17.077815486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:13:17.081227555Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=warn ts=2026-04-25T00:13:17.284015082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:19.530365616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:13:19.53457149Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-25T00:13:20.599712031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:13:20.603128343Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:13:21.648040259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:13:21.653070378Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-25T00:13:21.848481243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:13:21.851347555Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=warn ts=2026-04-25T00:13:22.284556022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:22.377152982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:13:22.383051394Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:13:24.73900257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:13:24.744560116Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:13:26.290296447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=error ts=2026-04-25T00:13:26.2968902Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=debug ts=2026-04-25T00:13:26.79201883Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:13:27.284724488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:28.604784978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:13:28.607656709Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" ts=2026-04-25T00:13:30.041889408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-adc48d01 10.129.0.74:7946" level=warn ts=2026-04-25T00:13:32.284348538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:32.505954423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:13:32.50884574Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:13:32.602735718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:13:32.605730953Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=debug ts=2026-04-25T00:13:34.631772848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-04-25T00:13:34.633395985Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-04-25T00:13:34.829993101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:13:34.832754165Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:13:35.160540377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:13:35.163538316Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:13:37.284467267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:13:37.284481916Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:13:37.28449958Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:13:37.284509208Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-25T00:13:40.367902565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:13:40.372943913Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:13:40.666891419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:13:40.670166002Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=warn ts=2026-04-25T00:13:42.284372233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:42.574764525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:13:42.577551293Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=warn ts=2026-04-25T00:13:47.284496714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:51.059087723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:13:51.067125487Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:13:52.284370066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:13:53.302728541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:13:53.309163876Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" ts=2026-04-25T00:13:54.493613817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:36052" level=warn ts=2026-04-25T00:13:57.284129605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:13:59.946498331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43580" ts=2026-04-25T00:14:00.044138511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946" level=debug ts=2026-04-25T00:14:02.25500785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=warn ts=2026-04-25T00:14:02.284123625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:02.524128089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:14:02.527390651Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=info ts=2026-04-25T00:14:03.102849405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:14:03.109981224Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:14:05.003543787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:14:05.006839939Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:14:06.244158442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:14:06.24753995Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=debug ts=2026-04-25T00:14:06.992910677Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.49:11211 count=1 level=info ts=2026-04-25T00:14:06.992944889Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=info ts=2026-04-25T00:14:07.283769227Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:14:07.283798363Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:14:07.283807735Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:14:07.283830366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:07.525926208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:14:07.530985525Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-25T00:14:11.485892419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:14:11.491571127Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-25T00:14:12.198389421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:14:12.20481954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:14:12.283918657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:15.464359293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=error ts=2026-04-25T00:14:15.47166838Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=warn ts=2026-04-25T00:14:17.284255497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:18.949311024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:14:18.953519463Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:14:19.29898076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-25T00:14:20.841573331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:14:20.84474013Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:14:22.284771238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:22.864227338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:14:22.867603488Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:14:23.742994402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:14:23.74622412Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" ts=2026-04-25T00:14:24.494847844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:45854" level=info ts=2026-04-25T00:14:25.23776698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:14:25.241019043Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:14:27.28441572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:28.4495343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:14:28.452448358Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" ts=2026-04-25T00:14:30.046308051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946" level=info ts=2026-04-25T00:14:30.988920135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:14:30.992746609Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=warn ts=2026-04-25T00:14:32.284565225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:14:34.394292998Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:14:34.865094532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:14:34.868253722Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:14:35.662793182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:14:35.666408206Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-25T00:14:37.284295116Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:14:37.284330998Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:14:37.284339649Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:14:37.284368371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-25T00:14:42.283809662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:46.771250129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:14:46.779513329Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:14:47.284437606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:47.954441381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:14:47.958195634Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=info ts=2026-04-25T00:14:48.140778268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:14:48.146895193Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" level=warn ts=2026-04-25T00:14:52.283942933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:54.547473493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:14:54.55160608Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-25T00:14:56.229342254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:14:56.232115166Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:14:56.895711989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:14:56.902074921Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:14:57.284251825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:14:57.964974468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:14:57.970309191Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" ts=2026-04-25T00:14:58.064330346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:33944" level=info ts=2026-04-25T00:14:58.074700112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:14:58.077803585Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" ts=2026-04-25T00:14:59.949691919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46556" ts=2026-04-25T00:15:00.048288259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946" level=info ts=2026-04-25T00:15:01.400168805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:15:01.403153229Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:15:02.28456478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:05.892539112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-25T00:15:06.992945642Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:15:06.993467919Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.49:11211 count=1 level=debug ts=2026-04-25T00:15:06.994812895Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads" level=debug ts=2026-04-25T00:15:06.994882815Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974528 n=1 level=debug ts=2026-04-25T00:15:06.994942867Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974528 n=1 level=debug ts=2026-04-25T00:15:07.109478898Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:15:07.284107703Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:15:07.28414366Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:15:07.284154555Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:15:07.284152271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:15:08.361315259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:15:08.36539582Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:15:10.47844722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=error ts=2026-04-25T00:15:10.486017821Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=info ts=2026-04-25T00:15:10.942256851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:15:10.94854454Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:15:12.2842361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=warn ts=2026-04-25T00:15:17.284303749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:15:17.595495756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:15:17.600364037Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:15:17.948444184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:15:17.952036772Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:15:21.139637036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:15:21.143260067Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:15:21.325824533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:15:21.32865534Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=info ts=2026-04-25T00:15:21.606568408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:15:21.609670111Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=warn ts=2026-04-25T00:15:22.284161154Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:15:23.66663236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:15:23.672544163Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:15:26.04406986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:15:26.047656054Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=warn ts=2026-04-25T00:15:27.28430718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:15:30.050298999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946" level=warn ts=2026-04-25T00:15:32.284367423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:15:34.569740077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:15:34.576684649Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" level=info ts=2026-04-25T00:15:37.283901999Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:15:37.28390388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:15:37.283933029Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:15:37.283942056Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:15:38.267594449Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:15:41.142471514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:15:41.148987412Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:15:41.955847812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:15:41.95977266Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=warn ts=2026-04-25T00:15:42.283821177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:15:43.690926537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:15:43.694627667Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=info ts=2026-04-25T00:15:46.538983663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:15:46.542664682Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:15:46.753180249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="28 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:15:46.759757894Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=warn ts=2026-04-25T00:15:47.284483589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:15:49.069378598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:15:49.073801556Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=info ts=2026-04-25T00:15:50.632861891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:15:50.636016461Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:15:52.129084256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:15:52.13487396Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=warn ts=2026-04-25T00:15:52.284312861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:15:53.921541509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:15:53.930250607Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=warn ts=2026-04-25T00:15:57.283918621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" ts=2026-04-25T00:16:00.052359715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946" level=warn ts=2026-04-25T00:16:02.284483684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:16:04.277749616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:16:04.282203668Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=info ts=2026-04-25T00:16:06.993116827Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:16:06.993124824Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.49:11211 count=1 level=info ts=2026-04-25T00:16:07.28462692Z caller=checkpoint.go:612 msg="starting checkpoint" level=warn ts=2026-04-25T00:16:07.284622455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:16:07.284655554Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:16:07.284673892Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:16:07.284691884Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=info ts=2026-04-25T00:16:07.284787258Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000022 level=debug ts=2026-04-25T00:16:07.284972289Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB" level=debug ts=2026-04-25T00:16:09.542397692Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:16:09.589626633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=error ts=2026-04-25T00:16:09.596744466Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=debug ts=2026-04-25T00:16:10.618638854Z caller=checkpoint.go:378 msg="writing series" size="265 kB" level=info ts=2026-04-25T00:16:10.666245893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:16:10.668324021Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:16:11.086576378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:16:11.089609174Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:16:11.339536594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:16:11.342792165Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:16:11.772934472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:16:11.775830624Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-25T00:16:12.283977244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:13.952458112Z caller=checkpoint.go:378 msg="writing series" size="78 kB" level=info ts=2026-04-25T00:16:15.159550185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:16:15.164871545Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" level=info ts=2026-04-25T00:16:16.011444709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:16:16.014221443Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=debug ts=2026-04-25T00:16:17.190008791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-04-25T00:16:17.284313182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:17.285486753Z caller=checkpoint.go:378 msg="writing series" size="52 kB" level=info ts=2026-04-25T00:16:19.729389599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:16:19.733203621Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=debug ts=2026-04-25T00:16:20.6190706Z caller=checkpoint.go:378 msg="writing series" size="275 kB" level=warn ts=2026-04-25T00:16:22.283833753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:16:23.219303202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:16:23.223036501Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=debug ts=2026-04-25T00:16:23.951959007Z caller=checkpoint.go:378 msg="writing series" size="10 kB" level=info ts=2026-04-25T00:16:26.787919345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:16:26.791401535Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=warn ts=2026-04-25T00:16:27.28474816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:27.286557775Z caller=checkpoint.go:378 msg="writing series" size="635 kB" level=debug ts=2026-04-25T00:16:27.286575653Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7 level=info ts=2026-04-25T00:16:29.39947548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:16:29.402884663Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" ts=2026-04-25T00:16:29.954382345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39008" ts=2026-04-25T00:16:30.053630637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946" level=debug ts=2026-04-25T00:16:30.618607174Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB" level=info ts=2026-04-25T00:16:30.769965076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:16:30.775244014Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=warn ts=2026-04-25T00:16:32.284488296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:16:32.938222717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:16:32.942146905Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:16:33.559767345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="28 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:16:33.566050652Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:16:33.952203688Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB" level=debug ts=2026-04-25T00:16:34.873230478Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:16:36.763279081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:16:36.769264329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:16:37.284231057Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:16:37.284259087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:37.284268089Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:16:37.284279591Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:16:37.285560068Z caller=checkpoint.go:378 msg="writing series" size="193 kB" level=info ts=2026-04-25T00:16:39.347723114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:16:39.354408495Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-25T00:16:40.618487155Z caller=checkpoint.go:378 msg="writing series" size="658 B" level=warn ts=2026-04-25T00:16:42.284434652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:43.952670664Z caller=checkpoint.go:378 msg="writing series" size="193 kB" level=info ts=2026-04-25T00:16:47.007704489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="799 B" avg_comp="799 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:16:47.011239193Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=warn ts=2026-04-25T00:16:47.284363741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:47.285545983Z caller=checkpoint.go:378 msg="writing series" size="121 kB" level=info ts=2026-04-25T00:16:49.865606485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:16:49.87217535Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=debug ts=2026-04-25T00:16:50.618447972Z caller=checkpoint.go:378 msg="writing series" size="264 kB" level=warn ts=2026-04-25T00:16:52.283975986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:16:53.633702091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:16:53.637441038Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:16:53.952916135Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB" level=debug ts=2026-04-25T00:16:53.952939429Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=8 ts=2026-04-25T00:16:54.504096994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:41122" level=info ts=2026-04-25T00:16:57.134035038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:16:57.137353835Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=warn ts=2026-04-25T00:16:57.284300097Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:16:57.285489354Z caller=checkpoint.go:378 msg="writing series" size="195 kB" ts=2026-04-25T00:16:59.956305787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49804" ts=2026-04-25T00:17:00.054492902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-adc48d01 10.129.0.74:7946" level=debug ts=2026-04-25T00:17:00.619107834Z caller=checkpoint.go:378 msg="writing series" size="343 kB" level=info ts=2026-04-25T00:17:01.780667439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:17:01.783543951Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:17:02.199477714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:17:02.2023705Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=warn ts=2026-04-25T00:17:02.284474988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:03.952612892Z caller=checkpoint.go:378 msg="writing series" size="654 kB" level=debug ts=2026-04-25T00:17:03.952636936Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3 level=debug ts=2026-04-25T00:17:05.131641763Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:17:06.993039906Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:17:06.993508441Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.49:11211 count=1 level=info ts=2026-04-25T00:17:07.284005364Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:17:07.28401942Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:07.284036689Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:17:07.284049411Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:17:07.285520012Z caller=checkpoint.go:378 msg="writing series" size="667 kB" level=info ts=2026-04-25T00:17:07.329342136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=error ts=2026-04-25T00:17:07.335981667Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=debug ts=2026-04-25T00:17:10.618929161Z caller=checkpoint.go:378 msg="writing series" size="359 kB" level=info ts=2026-04-25T00:17:11.058090911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:17:11.061569741Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-25T00:17:11.399339531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-04-25T00:17:12.284535723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:17:13.350893907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:17:13.354032312Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:17:13.921769065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:17:13.927781543Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" level=debug ts=2026-04-25T00:17:13.951893353Z caller=checkpoint.go:378 msg="writing series" size="42 kB" level=debug ts=2026-04-25T00:17:13.951916125Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3 level=info ts=2026-04-25T00:17:16.224874539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:17:16.228820208Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=info ts=2026-04-25T00:17:16.878785114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:17:16.883052879Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=warn ts=2026-04-25T00:17:17.284133236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:17.285217523Z caller=checkpoint.go:378 msg="writing series" size="60 kB" level=info ts=2026-04-25T00:17:18.694910207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:17:18.701148457Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:17:19.1043936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:17:19.107548324Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=debug ts=2026-04-25T00:17:20.619239118Z caller=checkpoint.go:378 msg="writing series" size="583 kB" level=warn ts=2026-04-25T00:17:22.284200412Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:17:23.444540853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:17:23.447811909Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=debug ts=2026-04-25T00:17:23.952237558Z caller=checkpoint.go:378 msg="writing series" size="13 kB" level=info ts=2026-04-25T00:17:24.984926692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:17:24.989090721Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=warn ts=2026-04-25T00:17:27.284388421Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:27.285561132Z caller=checkpoint.go:378 msg="writing series" size="355 kB" level=debug ts=2026-04-25T00:17:29.439191579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:17:29.525387113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:17:29.528849641Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" ts=2026-04-25T00:17:29.957809918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42046" ts=2026-04-25T00:17:30.056032288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-adc48d01 10.129.0.74:7946" level=debug ts=2026-04-25T00:17:30.618821737Z caller=checkpoint.go:378 msg="writing series" size="23 kB" level=info ts=2026-04-25T00:17:31.93090089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:17:31.934298042Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-25T00:17:32.284575023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:17:32.447299832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="28 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:17:32.453751275Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:17:33.795203382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:17:33.802227804Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=debug ts=2026-04-25T00:17:33.952483857Z caller=checkpoint.go:378 msg="writing series" size="26 kB" level=debug ts=2026-04-25T00:17:33.952509091Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6 level=info ts=2026-04-25T00:17:37.284474039Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:17:37.284489129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:37.284510143Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:17:37.284520092Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:17:37.294345778Z caller=checkpoint.go:378 msg="writing series" size="12 MB" level=debug ts=2026-04-25T00:17:37.294366965Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=1 level=info ts=2026-04-25T00:17:38.622443097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:17:38.626367443Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:17:40.621445538Z caller=checkpoint.go:378 msg="writing series" size="3.3 MB" level=debug ts=2026-04-25T00:17:40.62147143Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.3 MB" series=1 level=warn ts=2026-04-25T00:17:42.284231822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:43.952571176Z caller=checkpoint.go:378 msg="writing series" size="55 kB" level=warn ts=2026-04-25T00:17:47.28391307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:47.285176719Z caller=checkpoint.go:378 msg="writing series" size="416 kB" level=info ts=2026-04-25T00:17:48.554385487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:17:48.559358556Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-25T00:17:48.672808232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=error ts=2026-04-25T00:17:48.679528178Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=debug ts=2026-04-25T00:17:50.619270048Z caller=checkpoint.go:378 msg="writing series" size="128 kB" level=warn ts=2026-04-25T00:17:52.284489915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:53.95231837Z caller=checkpoint.go:378 msg="writing series" size="113 kB" level=info ts=2026-04-25T00:17:54.03169623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:17:54.034817495Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:17:54.722624811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:17:54.725500576Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:17:56.270994522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:17:56.274590874Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:17:57.284394399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:17:57.285512255Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB" level=info ts=2026-04-25T00:17:58.512951977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:17:58.516001899Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:17:59.274325885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:17:59.281138849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:17:59.281162624Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:17:59.281190022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:17:59.2869635Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" ts=2026-04-25T00:18:00.057238873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-adc48d01 10.129.0.74:7946" level=debug ts=2026-04-25T00:18:00.626986344Z caller=checkpoint.go:378 msg="writing series" size="14 MB" level=debug ts=2026-04-25T00:18:00.627011723Z caller=checkpoint.go:390 msg="flushing series" totalSize="15 MB" series=6 level=info ts=2026-04-25T00:18:00.834679742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3ba22b8046cbcd92 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:18:00.840593386Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3ba22b8046cbcd92/19dc1b66017%3A19dc1dd4138%3A8489fdee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-utih\"}" level=error ts=2026-04-25T00:18:00.840616231Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:00.840640366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:18:00.84305818Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a2b8c2%3A19dc1a5f192%3Ae42db2df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=debug ts=2026-04-25T00:18:01.747596518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-04-25T00:18:01.74949146Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=warn ts=2026-04-25T00:18:02.284477653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:02.573390903Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=debug ts=2026-04-25T00:18:03.952695223Z caller=checkpoint.go:378 msg="writing series" size="280 kB" level=info ts=2026-04-25T00:18:06.99302499Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:18:06.993042342Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.49:11211 count=1 level=info ts=2026-04-25T00:18:07.19882925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:18:07.201783182Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:18:07.283961419Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=warn ts=2026-04-25T00:18:07.283974636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:07.283989794Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:18:07.283999088Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:18:07.285379606Z caller=checkpoint.go:378 msg="writing series" size="637 kB" level=debug ts=2026-04-25T00:18:10.61908647Z caller=checkpoint.go:378 msg="writing series" size="15 kB" level=info ts=2026-04-25T00:18:11.021274235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:18:11.027308845Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=info ts=2026-04-25T00:18:11.104534915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"integration-service\"}" level=error ts=2026-04-25T00:18:11.108578884Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dc1a34c37%3A19dc1d6cce0%3A91b5ba91?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}" level=error ts=2026-04-25T00:18:11.108600575Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:11.108625193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="800 kB" avg_comp="800 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:18:11.112091221Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a2b462%3A19dc1f1ee30%3A675c02bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-25T00:18:12.270126504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:18:12.273821796Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=warn ts=2026-04-25T00:18:12.28392809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:13.952454211Z caller=checkpoint.go:378 msg="writing series" size="227 kB" level=debug ts=2026-04-25T00:18:13.952479642Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4 level=info ts=2026-04-25T00:18:14.77392887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:18:14.778700005Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a2b8c2%3A19dc1a5f192%3Ae42db2df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:18:16.082261213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="28 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:18:16.088634705Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:18:16.088655391Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:16.088677201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="28 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:18:16.094311477Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:18:16.451362044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:18:16.453940479Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=warn ts=2026-04-25T00:18:17.284338264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:17.287185958Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB" level=debug ts=2026-04-25T00:18:17.287208419Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=1 level=info ts=2026-04-25T00:18:19.581682977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:18:19.586376735Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:18:19.586404876Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:19.586454627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:18:19.589523054Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=debug ts=2026-04-25T00:18:20.619090233Z caller=checkpoint.go:378 msg="writing series" size="77 kB" level=info ts=2026-04-25T00:18:20.99211534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:18:20.999085434Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:18:20.999111765Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:20.999139068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:18:21.004480922Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:18:21.844177726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="800 kB" avg_comp="800 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:18:21.848723548Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a2b462%3A19dc1f1ee30%3A675c02bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=warn ts=2026-04-25T00:18:22.28386124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:23.03283947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:18:23.037478794Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:18:23.037505445Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:23.037529598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:18:23.040130048Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=debug ts=2026-04-25T00:18:23.95172909Z caller=checkpoint.go:378 msg="writing series" size="13 kB" level=info ts=2026-04-25T00:18:24.632719465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:18:24.635612816Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:18:24.635633554Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:24.635655574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:18:24.637753071Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:18:25.371228769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:18:25.374685182Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:18:25.374706356Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:25.374727962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:18:25.377464483Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=warn ts=2026-04-25T00:18:27.283867363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:27.286026328Z caller=checkpoint.go:378 msg="writing series" size="188 kB" level=info ts=2026-04-25T00:18:29.026606447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:18:29.031298164Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:18:29.031322369Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:29.031345877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:18:29.033826307Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" ts=2026-04-25T00:18:30.058803282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-adc48d01 10.129.0.74:7946" level=debug ts=2026-04-25T00:18:30.618941025Z caller=checkpoint.go:378 msg="writing series" size="56 kB" level=warn ts=2026-04-25T00:18:32.284263952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:33.952245102Z caller=checkpoint.go:378 msg="writing series" size="177 kB" level=info ts=2026-04-25T00:18:34.539966506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:18:34.544166864Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=info ts=2026-04-25T00:18:34.806028943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="28 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:18:34.812320826Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=info ts=2026-04-25T00:18:35.422875206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:18:35.428670558Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=error ts=2026-04-25T00:18:35.428695074Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:35.428722366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:18:35.433945994Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-25T00:18:36.93853938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:18:36.944358495Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=warn ts=2026-04-25T00:18:37.283893163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:37.283886657Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:18:37.283921772Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:18:37.283930069Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=debug ts=2026-04-25T00:18:37.285006374Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB" level=info ts=2026-04-25T00:18:37.531877366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:18:37.535057102Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=debug ts=2026-04-25T00:18:38.184795576Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:18:40.382023687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:18:40.384861169Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:18:40.477259852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:18:40.480117041Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:18:40.480140563Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:40.480162794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:18:40.482389386Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=debug ts=2026-04-25T00:18:40.619128221Z caller=checkpoint.go:378 msg="writing series" size="120 kB" level=info ts=2026-04-25T00:18:41.823464747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:18:41.826354043Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=warn ts=2026-04-25T00:18:42.28438062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:43.701714554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:18:43.704998186Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:18:43.952906354Z caller=checkpoint.go:378 msg="writing series" size="742 kB" level=debug ts=2026-04-25T00:18:43.952931714Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=8 level=info ts=2026-04-25T00:18:44.913695811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:18:44.916567183Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:18:44.916588574Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:44.916609723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:18:44.91917138Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:18:45.416655202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="800 kB" avg_comp="800 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"openshift-multus\"}" level=error ts=2026-04-25T00:18:45.420366542Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dc1a2b462%3A19dc1f1ee30%3A675c02bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}" level=info ts=2026-04-25T00:18:46.157342277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f3cdaad98662efb immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-lcho\"}" level=error ts=2026-04-25T00:18:46.163671346Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f3cdaad98662efb/19dc1b6e8d0%3A19dc1c8c12a%3A5e30af7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-lcho\"}" level=error ts=2026-04-25T00:18:46.163695116Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:46.163720199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:18:46.166530162Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a2b69c%3A19dc1f8457b%3Aef445032?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=warn ts=2026-04-25T00:18:47.284386869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:47.285524271Z caller=checkpoint.go:378 msg="writing series" size="138 kB" level=info ts=2026-04-25T00:18:47.898543624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:18:47.903839886Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=info ts=2026-04-25T00:18:48.943115337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="155 kB" avg_uncomp="155 kB" full=1 labels="{stream=\"openshift-console-operator\"}" level=error ts=2026-04-25T00:18:48.946141189Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dc1a2b8c2%3A19dc1a5f192%3Ae42db2df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}" level=info ts=2026-04-25T00:18:49.767559987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:18:49.770537708Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:18:49.770562415Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:49.770585188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:18:49.772790072Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:18:50.207386322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="118 MB" avg_uncomp="20 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}" level=error ts=2026-04-25T00:18:50.213186611Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dc1a2b951%3A19dc1a43f89%3Adef84e1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}" level=debug ts=2026-04-25T00:18:50.618340966Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB" level=warn ts=2026-04-25T00:18:52.284084741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:18:52.946351584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:18:52.949820453Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=error ts=2026-04-25T00:18:52.949839764Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:52.949860782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:18:52.952174547Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:18:53.847799974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:18:53.850543534Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:18:53.850563538Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:18:53.850582721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:18:53.852756657Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=debug ts=2026-04-25T00:18:53.952354821Z caller=checkpoint.go:378 msg="writing series" size="602 kB" ts=2026-04-25T00:18:54.510876412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:32886" level=debug ts=2026-04-25T00:18:54.744008259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=warn ts=2026-04-25T00:18:57.284240484Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:18:57.285288241Z caller=checkpoint.go:378 msg="writing series" size="20 kB" level=info ts=2026-04-25T00:18:58.91963557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:18:58.923041022Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" ts=2026-04-25T00:19:00.060113475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-adc48d01 10.129.0.74:7946" level=debug ts=2026-04-25T00:19:00.619854179Z caller=checkpoint.go:378 msg="writing series" size="2.6 MB" level=debug ts=2026-04-25T00:19:00.619877312Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.4 MB" series=5 level=info ts=2026-04-25T00:19:01.440253534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}" level=error ts=2026-04-25T00:19:01.444732586Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19dc1a2b69c%3A19dc1f8457b%3Aef445032?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}" level=info ts=2026-04-25T00:19:01.530972061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:19:01.534121765Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=info ts=2026-04-25T00:19:02.084494286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:19:02.089009087Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:19:02.08904032Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:19:02.089076841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:19:02.092704046Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=warn ts=2026-04-25T00:19:02.284219866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:19:03.102099066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="28 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}" level=error ts=2026-04-25T00:19:03.108713168Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dc1a2b462%3A19dc1c6362c%3A5a690ca6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}" level=debug ts=2026-04-25T00:19:03.952238233Z caller=checkpoint.go:378 msg="writing series" size="187 kB" level=debug ts=2026-04-25T00:19:04.09295255Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335 level=info ts=2026-04-25T00:19:04.395832349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="461 kB" avg_comp="461 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}" level=error ts=2026-04-25T00:19:04.399954556Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dc1a2b886%3A19dc1f8a8d9%3A3cc25a46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}" level=info ts=2026-04-25T00:19:04.646543872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"product-kubearchive\"}" level=error ts=2026-04-25T00:19:04.652195966Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dc1a34c36%3A19dc1d1cf88%3Ad661efd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}" level=info ts=2026-04-25T00:19:06.992967797Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables" level=debug ts=2026-04-25T00:19:06.993208852Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.49:11211 count=1 level=info ts=2026-04-25T00:19:07.283751539Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job" level=debug ts=2026-04-25T00:19:07.283783937Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job" level=info ts=2026-04-25T00:19:07.283796161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job" level=warn ts=2026-04-25T00:19:07.283831607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:07.285936927Z caller=checkpoint.go:378 msg="writing series" size="935 B" level=info ts=2026-04-25T00:19:09.164164493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}" level=error ts=2026-04-25T00:19:09.167523748Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dc1a2dfe7%3A19dc1a33793%3A485250e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}" level=info ts=2026-04-25T00:19:09.363026599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}" level=error ts=2026-04-25T00:19:09.366573017Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dc1a329aa%3A19dc1a36ebc%3Aac2264ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}" level=info ts=2026-04-25T00:19:09.673472562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eaaa4255bb2d341c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:19:09.676747808Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eaaa4255bb2d341c/19dc1b5cf90%3A19dc1bec50a%3Aaef086f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kszx\"}" level=error ts=2026-04-25T00:19:09.676768569Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries" level=info ts=2026-04-25T00:19:09.676791235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:19:09.679131574Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a33792%3A19dc1a33793%3A3215d89e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=debug ts=2026-04-25T00:19:10.619066811Z caller=checkpoint.go:378 msg="writing series" size="101 kB" level=info ts=2026-04-25T00:19:11.098410766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}" level=error ts=2026-04-25T00:19:11.10150534Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dc1a329a9%3A19dc1a3dc1b%3A9914c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}" level=info ts=2026-04-25T00:19:11.454475928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="817 B" avg_comp="817 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"konflux-policies\"}" level=error ts=2026-04-25T00:19:11.457631399Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dc1a337b5%3A19dc1a4a58a%3Acbe63563?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}" level=info ts=2026-04-25T00:19:12.001213905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}" level=error ts=2026-04-25T00:19:12.003840717Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dc1a3376d%3A19dc1a33793%3A247bf1b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}" level=info ts=2026-04-25T00:19:12.270762464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a46bcf5975818527 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-xkgv\"}" level=error ts=2026-04-25T00:19:12.27362779Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a46bcf5975818527/19dc1b769b7%3A19dc1b91fb9%3Ac1582956?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xkgv\"}" level=warn ts=2026-04-25T00:19:12.284713193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:13.952326393Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB" level=warn ts=2026-04-25T00:19:17.284688568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:17.286309452Z caller=checkpoint.go:378 msg="writing series" size="862 kB" level=debug ts=2026-04-25T00:19:17.286328904Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5 level=info ts=2026-04-25T00:19:19.991401506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}" level=error ts=2026-04-25T00:19:19.995160232Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dc1a2b82c%3A19dc1c74056%3A261e1580?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}" level=debug ts=2026-04-25T00:19:20.619011663Z caller=checkpoint.go:378 msg="writing series" size="565 kB" level=info ts=2026-04-25T00:19:21.202238489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}" level=error ts=2026-04-25T00:19:21.20540959Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dc1a33792%3A19dc1a33793%3A3215d89e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}" level=info ts=2026-04-25T00:19:21.624481297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}" level=error ts=2026-04-25T00:19:21.627543363Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dc1a2b657%3A19dc1a33793%3Adb0eaffa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}" level=info ts=2026-04-25T00:19:21.929279186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="472 kB" avg_comp="472 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}" level=error ts=2026-04-25T00:19:21.933124291Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dc1a2b718%3A19dc1f259dc%3A93a60154?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}" level=warn ts=2026-04-25T00:19:22.284137292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=info ts=2026-04-25T00:19:23.303713836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"tekton-results\"}" level=error ts=2026-04-25T00:19:23.310814394Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19dc1a329a9%3A19dc1ce2e11%3A458ed36e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}" level=debug ts=2026-04-25T00:19:23.952496075Z caller=checkpoint.go:378 msg="writing series" size="10 kB" level=info ts=2026-04-25T00:19:25.299856282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}" level=error ts=2026-04-25T00:19:25.302963597Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dc1b640d8%3A19dc1b78b2b%3Ac8db42cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}" level=warn ts=2026-04-25T00:19:27.284111049Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round" level=debug ts=2026-04-25T00:19:27.285182337Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB" ts=2026-04-25T00:19:30.061759694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946" level=debug ts=2026-04-25T00:19:30.63349468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"